2024-11-13 14:29:20,660 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 14:29:20,678 main DEBUG Took 0.011362 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 14:29:20,679 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 14:29:20,679 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 14:29:20,681 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 14:29:20,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,691 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 14:29:20,704 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,706 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,708 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,709 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,711 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,712 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,714 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,715 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,716 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,716 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,717 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,718 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,719 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:29:20,720 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,721 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 14:29:20,723 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:29:20,725 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 14:29:20,728 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 14:29:20,729 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 14:29:20,730 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 14:29:20,731 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 14:29:20,743 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 14:29:20,747 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 14:29:20,749 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 14:29:20,749 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 14:29:20,750 main DEBUG createAppenders(={Console}) 2024-11-13 14:29:20,751 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-13 14:29:20,751 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 14:29:20,752 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-13 14:29:20,752 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 14:29:20,753 main DEBUG OutputStream closed 2024-11-13 14:29:20,753 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 14:29:20,753 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 14:29:20,754 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-13 14:29:20,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 14:29:20,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 14:29:20,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 14:29:20,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 14:29:20,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 14:29:20,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 14:29:20,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 14:29:20,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 14:29:20,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 14:29:20,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 14:29:20,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 14:29:20,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 14:29:20,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 14:29:20,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 14:29:20,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 14:29:20,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 14:29:20,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 14:29:20,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 14:29:20,844 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 14:29:20,844 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-13 14:29:20,844 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 14:29:20,845 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-13T14:29:20,856 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-13 14:29:20,859 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 14:29:20,859 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T14:29:20,907 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=692, ProcessCount=11, AvailableMemoryMB=4168 2024-11-13T14:29:21,119 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9 2024-11-13T14:29:21,623 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T14:29:21,715 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T14:29:21,773 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-13T14:29:21,826 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508161815 2024-11-13T14:29:21,891 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508161815 2024-11-13T14:29:21,955 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:21,960 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-13T14:29:21,961 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-13T14:29:21,993 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => daa2ca368b2108264a9146503bae95e1, NAME => 'testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9 2024-11-13T14:29:22,045 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T14:29:22,049 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:29:22,120 INFO [StoreOpener-daa2ca368b2108264a9146503bae95e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region daa2ca368b2108264a9146503bae95e1 2024-11-13T14:29:22,149 INFO [StoreOpener-daa2ca368b2108264a9146503bae95e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daa2ca368b2108264a9146503bae95e1 columnFamilyName MyCF 2024-11-13T14:29:22,155 DEBUG [StoreOpener-daa2ca368b2108264a9146503bae95e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:29:22,162 INFO [StoreOpener-daa2ca368b2108264a9146503bae95e1-1 {}] regionserver.HStore(327): Store=daa2ca368b2108264a9146503bae95e1/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:29:22,167 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for daa2ca368b2108264a9146503bae95e1 2024-11-13T14:29:22,174 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/data/default/testLockupAroundBadAssignSync/daa2ca368b2108264a9146503bae95e1 2024-11-13T14:29:22,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/data/default/testLockupAroundBadAssignSync/daa2ca368b2108264a9146503bae95e1 2024-11-13T14:29:22,178 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for daa2ca368b2108264a9146503bae95e1 2024-11-13T14:29:22,179 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for daa2ca368b2108264a9146503bae95e1 2024-11-13T14:29:22,187 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for daa2ca368b2108264a9146503bae95e1 2024-11-13T14:29:22,198 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/data/default/testLockupAroundBadAssignSync/daa2ca368b2108264a9146503bae95e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:29:22,200 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened daa2ca368b2108264a9146503bae95e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63412035, jitterRate=-0.05508704483509064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:29:22,208 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T14:29:22,222 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for daa2ca368b2108264a9146503bae95e1: Writing region info on filesystem at 1731508162093Initializing all the Stores at 1731508162094 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1731508162095 (+1 ms)Cleaning up temporary data from old regions at 1731508162180 (+85 ms)Region opened successfully at 1731508162220 (+40 ms) 2024-11-13T14:29:22,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508161815) roll requested 2024-11-13T14:29:22,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162266 2024-11-13T14:29:22,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508161815 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162266 2024-11-13T14:29:22,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162266) roll requested 2024-11-13T14:29:22,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162278 2024-11-13T14:29:22,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162278 2024-11-13T14:29:22,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162278) roll requested 2024-11-13T14:29:22,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162297 2024-11-13T14:29:22,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162297 2024-11-13T14:29:22,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162297) roll requested 2024-11-13T14:29:22,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162310 2024-11-13T14:29:22,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162310 2024-11-13T14:29:22,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162310) roll requested 2024-11-13T14:29:22,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162324 2024-11-13T14:29:22,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162324 2024-11-13T14:29:22,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162324) roll requested 2024-11-13T14:29:22,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162334 2024-11-13T14:29:22,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162334 2024-11-13T14:29:22,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162334) roll requested 2024-11-13T14:29:22,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162347 2024-11-13T14:29:22,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162347 2024-11-13T14:29:22,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162347) roll requested 2024-11-13T14:29:22,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162358 2024-11-13T14:29:22,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162358 2024-11-13T14:29:22,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162358) roll requested 2024-11-13T14:29:22,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162367 2024-11-13T14:29:22,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162367 2024-11-13T14:29:22,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162367) roll requested 2024-11-13T14:29:22,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162376 2024-11-13T14:29:22,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162376 2024-11-13T14:29:22,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162376) roll requested 2024-11-13T14:29:22,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162388 2024-11-13T14:29:22,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162388 2024-11-13T14:29:22,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162388) roll requested 2024-11-13T14:29:22,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162401 2024-11-13T14:29:22,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162401 2024-11-13T14:29:22,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162401) roll requested 2024-11-13T14:29:22,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162411 2024-11-13T14:29:22,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162411 2024-11-13T14:29:22,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162411) roll requested 2024-11-13T14:29:22,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162421 2024-11-13T14:29:22,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162421 2024-11-13T14:29:22,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162421) roll requested 2024-11-13T14:29:22,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162432 2024-11-13T14:29:22,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162432 2024-11-13T14:29:22,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162432) roll requested 2024-11-13T14:29:22,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162444 2024-11-13T14:29:22,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162444 2024-11-13T14:29:22,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162444) roll requested 2024-11-13T14:29:22,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162454 2024-11-13T14:29:22,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162454 2024-11-13T14:29:22,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162454) roll requested 2024-11-13T14:29:22,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162467 2024-11-13T14:29:22,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162467 2024-11-13T14:29:22,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162467) roll requested 2024-11-13T14:29:22,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162479 2024-11-13T14:29:22,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162479 2024-11-13T14:29:22,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162479) roll requested 2024-11-13T14:29:22,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162490 2024-11-13T14:29:22,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162490 2024-11-13T14:29:22,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162490) roll requested 2024-11-13T14:29:22,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162504 2024-11-13T14:29:22,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162504 2024-11-13T14:29:22,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162504) roll requested 2024-11-13T14:29:22,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162513 2024-11-13T14:29:22,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162513 2024-11-13T14:29:22,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162513) roll requested 2024-11-13T14:29:22,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162532 2024-11-13T14:29:22,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162532 2024-11-13T14:29:22,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162532) roll requested 2024-11-13T14:29:22,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162540 2024-11-13T14:29:22,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162540 2024-11-13T14:29:22,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162540) roll requested 2024-11-13T14:29:22,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162549 2024-11-13T14:29:22,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162549 2024-11-13T14:29:22,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162549) roll requested 2024-11-13T14:29:22,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162556 2024-11-13T14:29:22,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162556 2024-11-13T14:29:22,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162556) roll requested 2024-11-13T14:29:22,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162564 2024-11-13T14:29:22,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162564 2024-11-13T14:29:22,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162564) roll requested 2024-11-13T14:29:22,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162572 2024-11-13T14:29:22,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162572 2024-11-13T14:29:22,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162572) roll requested 2024-11-13T14:29:22,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162581 2024-11-13T14:29:22,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162581 2024-11-13T14:29:22,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162581) roll requested 2024-11-13T14:29:22,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162596 2024-11-13T14:29:22,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162596 2024-11-13T14:29:22,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162596) roll requested 2024-11-13T14:29:22,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162604 2024-11-13T14:29:22,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162604 2024-11-13T14:29:22,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162604) roll requested 2024-11-13T14:29:22,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162613 2024-11-13T14:29:22,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162613 2024-11-13T14:29:22,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162613) roll requested 2024-11-13T14:29:22,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162622 2024-11-13T14:29:22,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162622 2024-11-13T14:29:22,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162622) roll requested 2024-11-13T14:29:22,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162635 2024-11-13T14:29:22,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162635 2024-11-13T14:29:22,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162635) roll requested 2024-11-13T14:29:22,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162645 2024-11-13T14:29:22,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162645 2024-11-13T14:29:22,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162645) roll requested 2024-11-13T14:29:22,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162655 2024-11-13T14:29:22,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162655 2024-11-13T14:29:22,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162655) roll requested 2024-11-13T14:29:22,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162666 2024-11-13T14:29:22,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162666 2024-11-13T14:29:22,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162666) roll requested 2024-11-13T14:29:22,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162676 2024-11-13T14:29:22,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162676 2024-11-13T14:29:22,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162676) roll requested 2024-11-13T14:29:22,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162688 2024-11-13T14:29:22,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162688 2024-11-13T14:29:22,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162688) roll requested 2024-11-13T14:29:22,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162696 2024-11-13T14:29:22,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162696 2024-11-13T14:29:22,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162696) roll requested 2024-11-13T14:29:22,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162704 2024-11-13T14:29:22,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162704 2024-11-13T14:29:22,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162704) roll requested 2024-11-13T14:29:22,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162711 2024-11-13T14:29:22,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162711 2024-11-13T14:29:22,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162711) roll requested 2024-11-13T14:29:22,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162721 2024-11-13T14:29:22,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162721 2024-11-13T14:29:22,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162721) roll requested 2024-11-13T14:29:22,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162729 2024-11-13T14:29:22,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162729 2024-11-13T14:29:22,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162729) roll requested 2024-11-13T14:29:22,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162742 2024-11-13T14:29:22,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162742 2024-11-13T14:29:22,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162742) roll requested 2024-11-13T14:29:22,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162750 2024-11-13T14:29:22,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162750 2024-11-13T14:29:22,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162750) roll requested 2024-11-13T14:29:22,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162757 2024-11-13T14:29:22,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162757 2024-11-13T14:29:22,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162757) roll requested 2024-11-13T14:29:22,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162765 2024-11-13T14:29:22,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162765 2024-11-13T14:29:22,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162765) roll requested 2024-11-13T14:29:22,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162776 2024-11-13T14:29:22,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162776 2024-11-13T14:29:22,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162776) roll requested 2024-11-13T14:29:22,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162784 2024-11-13T14:29:22,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162784 2024-11-13T14:29:22,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162784) roll requested 2024-11-13T14:29:22,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162793 2024-11-13T14:29:22,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162793 2024-11-13T14:29:22,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162793) roll requested 2024-11-13T14:29:22,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162800 2024-11-13T14:29:22,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162800 2024-11-13T14:29:22,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162800) roll requested 2024-11-13T14:29:22,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162807 2024-11-13T14:29:22,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162807 2024-11-13T14:29:22,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162807) roll requested 2024-11-13T14:29:22,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162815 2024-11-13T14:29:22,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162815 2024-11-13T14:29:22,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162815) roll requested 2024-11-13T14:29:22,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162822 2024-11-13T14:29:22,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162822 2024-11-13T14:29:22,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162822) roll requested 2024-11-13T14:29:22,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162832 2024-11-13T14:29:22,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162832 2024-11-13T14:29:22,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162832) roll requested 2024-11-13T14:29:22,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162839 2024-11-13T14:29:22,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162839 2024-11-13T14:29:22,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162839) roll requested 2024-11-13T14:29:22,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162847 2024-11-13T14:29:22,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162847 2024-11-13T14:29:22,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162847) roll requested 2024-11-13T14:29:22,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162854 2024-11-13T14:29:22,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162854 2024-11-13T14:29:22,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162854) roll requested 2024-11-13T14:29:22,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162862 2024-11-13T14:29:22,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162862 2024-11-13T14:29:22,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162862) roll requested 2024-11-13T14:29:22,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162870 2024-11-13T14:29:22,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162870 2024-11-13T14:29:22,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162870) roll requested 2024-11-13T14:29:22,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162881 2024-11-13T14:29:22,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162881 2024-11-13T14:29:22,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162881) roll requested 2024-11-13T14:29:22,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162894 2024-11-13T14:29:22,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162894 2024-11-13T14:29:22,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162894) roll requested 2024-11-13T14:29:22,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162905 2024-11-13T14:29:22,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162905 2024-11-13T14:29:22,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162905) roll requested 2024-11-13T14:29:22,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162916 2024-11-13T14:29:22,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162916 2024-11-13T14:29:22,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162916) roll requested 2024-11-13T14:29:22,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162925 2024-11-13T14:29:22,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162925 2024-11-13T14:29:22,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162925) roll requested 2024-11-13T14:29:22,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162933 2024-11-13T14:29:22,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162933 2024-11-13T14:29:22,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162933) roll requested 2024-11-13T14:29:22,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162940 2024-11-13T14:29:22,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162940 2024-11-13T14:29:22,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162940) roll requested 2024-11-13T14:29:22,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162946 2024-11-13T14:29:22,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162946 2024-11-13T14:29:22,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162946) roll requested 2024-11-13T14:29:22,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162954 2024-11-13T14:29:22,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162954 2024-11-13T14:29:22,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162954) roll requested 2024-11-13T14:29:22,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162960 2024-11-13T14:29:22,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162960 2024-11-13T14:29:22,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162960) roll requested 2024-11-13T14:29:22,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162968 2024-11-13T14:29:22,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162968 2024-11-13T14:29:22,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162968) roll requested 2024-11-13T14:29:22,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162977 2024-11-13T14:29:22,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162977 2024-11-13T14:29:22,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162977) roll requested 2024-11-13T14:29:22,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162986 2024-11-13T14:29:22,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:22,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162986 2024-11-13T14:29:22,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:22,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:22,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162986) roll requested 2024-11-13T14:29:22,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508162998 2024-11-13T14:29:23,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162998 2024-11-13T14:29:23,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508162998) roll requested 2024-11-13T14:29:23,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163008 2024-11-13T14:29:23,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508162998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163008 2024-11-13T14:29:23,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163008) roll requested 2024-11-13T14:29:23,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163018 2024-11-13T14:29:23,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163018 2024-11-13T14:29:23,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163018) roll requested 2024-11-13T14:29:23,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163025 2024-11-13T14:29:23,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163025 2024-11-13T14:29:23,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163025) roll requested 2024-11-13T14:29:23,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163033 2024-11-13T14:29:23,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163033 2024-11-13T14:29:23,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163033) roll requested 2024-11-13T14:29:23,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163042 2024-11-13T14:29:23,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163042 2024-11-13T14:29:23,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163042) roll requested 2024-11-13T14:29:23,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163052 2024-11-13T14:29:23,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163052 2024-11-13T14:29:23,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163052) roll requested 2024-11-13T14:29:23,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163060 2024-11-13T14:29:23,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163060 2024-11-13T14:29:23,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163060) roll requested 2024-11-13T14:29:23,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163069 2024-11-13T14:29:23,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163069 2024-11-13T14:29:23,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163069) roll requested 2024-11-13T14:29:23,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163079 2024-11-13T14:29:23,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163079 2024-11-13T14:29:23,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163079) roll requested 2024-11-13T14:29:23,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163089 2024-11-13T14:29:23,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163089 2024-11-13T14:29:23,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163089) roll requested 2024-11-13T14:29:23,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163096 2024-11-13T14:29:23,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163096 2024-11-13T14:29:23,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163096) roll requested 2024-11-13T14:29:23,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163104 2024-11-13T14:29:23,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163104 2024-11-13T14:29:23,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163104) roll requested 2024-11-13T14:29:23,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163112 2024-11-13T14:29:23,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163112 2024-11-13T14:29:23,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163112) roll requested 2024-11-13T14:29:23,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163121 2024-11-13T14:29:23,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163121 2024-11-13T14:29:23,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163121) roll requested 2024-11-13T14:29:23,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163131 2024-11-13T14:29:23,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163131 2024-11-13T14:29:23,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163131) roll requested 2024-11-13T14:29:23,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163141 2024-11-13T14:29:23,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163141 2024-11-13T14:29:23,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163141) roll requested 2024-11-13T14:29:23,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163153 2024-11-13T14:29:23,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163153 2024-11-13T14:29:23,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163153) roll requested 2024-11-13T14:29:23,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163162 2024-11-13T14:29:23,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163162 2024-11-13T14:29:23,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163162) roll requested 2024-11-13T14:29:23,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163171 2024-11-13T14:29:23,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163171 2024-11-13T14:29:23,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163171) roll requested 2024-11-13T14:29:23,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163181 2024-11-13T14:29:23,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163181 2024-11-13T14:29:23,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163181) roll requested 2024-11-13T14:29:23,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163191 2024-11-13T14:29:23,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163191 2024-11-13T14:29:23,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163191) roll requested 2024-11-13T14:29:23,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163200 2024-11-13T14:29:23,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163200 2024-11-13T14:29:23,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163200) roll requested 2024-11-13T14:29:23,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163211 2024-11-13T14:29:23,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163211 2024-11-13T14:29:23,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163211) roll requested 2024-11-13T14:29:23,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163222 2024-11-13T14:29:23,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163222 2024-11-13T14:29:23,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163222) roll requested 2024-11-13T14:29:23,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163233 2024-11-13T14:29:23,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163233 2024-11-13T14:29:23,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163233) roll requested 2024-11-13T14:29:23,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163241 2024-11-13T14:29:23,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163241 2024-11-13T14:29:23,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163241) roll requested 2024-11-13T14:29:23,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163251 2024-11-13T14:29:23,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163251 2024-11-13T14:29:23,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163251) roll requested 2024-11-13T14:29:23,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163259 2024-11-13T14:29:23,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163259 2024-11-13T14:29:23,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163259) roll requested 2024-11-13T14:29:23,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163268 2024-11-13T14:29:23,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163268 2024-11-13T14:29:23,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163268) roll requested 2024-11-13T14:29:23,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163277 2024-11-13T14:29:23,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163277 2024-11-13T14:29:23,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163277) roll requested 2024-11-13T14:29:23,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163288 2024-11-13T14:29:23,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163288 2024-11-13T14:29:23,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163288) roll requested 2024-11-13T14:29:23,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163297 2024-11-13T14:29:23,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163297 2024-11-13T14:29:23,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163297) roll requested 2024-11-13T14:29:23,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163307 2024-11-13T14:29:23,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163307 2024-11-13T14:29:23,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163307) roll requested 2024-11-13T14:29:23,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163318 2024-11-13T14:29:23,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163318 2024-11-13T14:29:23,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163318) roll requested 2024-11-13T14:29:23,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163328 2024-11-13T14:29:23,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163328 2024-11-13T14:29:23,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163328) roll requested 2024-11-13T14:29:23,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163339 2024-11-13T14:29:23,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163339 2024-11-13T14:29:23,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163339) roll requested 2024-11-13T14:29:23,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163350 2024-11-13T14:29:23,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163350 2024-11-13T14:29:23,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163350) roll requested 2024-11-13T14:29:23,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163361 2024-11-13T14:29:23,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163361 2024-11-13T14:29:23,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163361) roll requested 2024-11-13T14:29:23,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163369 2024-11-13T14:29:23,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163369 2024-11-13T14:29:23,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163369) roll requested 2024-11-13T14:29:23,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163377 2024-11-13T14:29:23,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163377 2024-11-13T14:29:23,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163377) roll requested 2024-11-13T14:29:23,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163383 2024-11-13T14:29:23,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163383 2024-11-13T14:29:23,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163383) roll requested 2024-11-13T14:29:23,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163391 2024-11-13T14:29:23,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163391 2024-11-13T14:29:23,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163391) roll requested 2024-11-13T14:29:23,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163400 2024-11-13T14:29:23,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163400 2024-11-13T14:29:23,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163400) roll requested 2024-11-13T14:29:23,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163410 2024-11-13T14:29:23,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163410 2024-11-13T14:29:23,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163410) roll requested 2024-11-13T14:29:23,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163421 2024-11-13T14:29:23,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163421 2024-11-13T14:29:23,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163421) roll requested 2024-11-13T14:29:23,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163428 2024-11-13T14:29:23,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163428 2024-11-13T14:29:23,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163428) roll requested 2024-11-13T14:29:23,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163437 2024-11-13T14:29:23,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163437 2024-11-13T14:29:23,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163437) roll requested 2024-11-13T14:29:23,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163446 2024-11-13T14:29:23,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163446 2024-11-13T14:29:23,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163446) roll requested 2024-11-13T14:29:23,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163456 2024-11-13T14:29:23,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163456 2024-11-13T14:29:23,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163456) roll requested 2024-11-13T14:29:23,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163465 2024-11-13T14:29:23,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163465 2024-11-13T14:29:23,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163465) roll requested 2024-11-13T14:29:23,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163474 2024-11-13T14:29:23,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163474 2024-11-13T14:29:23,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163474) roll requested 2024-11-13T14:29:23,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163484 2024-11-13T14:29:23,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163484 2024-11-13T14:29:23,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163484) roll requested 2024-11-13T14:29:23,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163493 2024-11-13T14:29:23,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163493 2024-11-13T14:29:23,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163493) roll requested 2024-11-13T14:29:23,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163504 2024-11-13T14:29:23,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163504 2024-11-13T14:29:23,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163504) roll requested 2024-11-13T14:29:23,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163510 2024-11-13T14:29:23,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163510 2024-11-13T14:29:23,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163510) roll requested 2024-11-13T14:29:23,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163520 2024-11-13T14:29:23,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163520 2024-11-13T14:29:23,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163520) roll requested 2024-11-13T14:29:23,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163529 2024-11-13T14:29:23,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163529 2024-11-13T14:29:23,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163529) roll requested 2024-11-13T14:29:23,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163535 2024-11-13T14:29:23,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163535 2024-11-13T14:29:23,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163535) roll requested 2024-11-13T14:29:23,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163542 2024-11-13T14:29:23,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163542 2024-11-13T14:29:23,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163542) roll requested 2024-11-13T14:29:23,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163549 2024-11-13T14:29:23,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163549 2024-11-13T14:29:23,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163549) roll requested 2024-11-13T14:29:23,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163557 2024-11-13T14:29:23,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163557 2024-11-13T14:29:23,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163557) roll requested 2024-11-13T14:29:23,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163565 2024-11-13T14:29:23,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163565 2024-11-13T14:29:23,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163565) roll requested 2024-11-13T14:29:23,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163576 2024-11-13T14:29:23,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163576 2024-11-13T14:29:23,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163576) roll requested 2024-11-13T14:29:23,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163586 2024-11-13T14:29:23,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163586 2024-11-13T14:29:23,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163586) roll requested 2024-11-13T14:29:23,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163597 2024-11-13T14:29:23,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163597 2024-11-13T14:29:23,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163597) roll requested 2024-11-13T14:29:23,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163606 2024-11-13T14:29:23,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163606 2024-11-13T14:29:23,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163606) roll requested 2024-11-13T14:29:23,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163619 2024-11-13T14:29:23,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163619 2024-11-13T14:29:23,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163619) roll requested 2024-11-13T14:29:23,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163629 2024-11-13T14:29:23,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163629 2024-11-13T14:29:23,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163629) roll requested 2024-11-13T14:29:23,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163635 2024-11-13T14:29:23,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163635 2024-11-13T14:29:23,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163635) roll requested 2024-11-13T14:29:23,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163643 2024-11-13T14:29:23,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163643 2024-11-13T14:29:23,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163643) roll requested 2024-11-13T14:29:23,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163649 2024-11-13T14:29:23,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163649 2024-11-13T14:29:23,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163649) roll requested 2024-11-13T14:29:23,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163659 2024-11-13T14:29:23,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163659 2024-11-13T14:29:23,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163659) roll requested 2024-11-13T14:29:23,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163665 2024-11-13T14:29:23,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163665 2024-11-13T14:29:23,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163665) roll requested 2024-11-13T14:29:23,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163674 2024-11-13T14:29:23,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163674 2024-11-13T14:29:23,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163674) roll requested 2024-11-13T14:29:23,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163681 2024-11-13T14:29:23,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163681 2024-11-13T14:29:23,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163681) roll requested 2024-11-13T14:29:23,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163688 2024-11-13T14:29:23,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163688 2024-11-13T14:29:23,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163688) roll requested 2024-11-13T14:29:23,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163695 2024-11-13T14:29:23,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163695 2024-11-13T14:29:23,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163695) roll requested 2024-11-13T14:29:23,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163701 2024-11-13T14:29:23,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163701 2024-11-13T14:29:23,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163701) roll requested 2024-11-13T14:29:23,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163708 2024-11-13T14:29:23,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163708 2024-11-13T14:29:23,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163708) roll requested 2024-11-13T14:29:23,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163715 2024-11-13T14:29:23,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163715 2024-11-13T14:29:23,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163715) roll requested 2024-11-13T14:29:23,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163722 2024-11-13T14:29:23,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163722 2024-11-13T14:29:23,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163722) roll requested 2024-11-13T14:29:23,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163728 2024-11-13T14:29:23,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163728 2024-11-13T14:29:23,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163728) roll requested 2024-11-13T14:29:23,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163735 2024-11-13T14:29:23,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163735 2024-11-13T14:29:23,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163735) roll requested 2024-11-13T14:29:23,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163741 2024-11-13T14:29:23,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163741 2024-11-13T14:29:23,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163741) roll requested 2024-11-13T14:29:23,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163747 2024-11-13T14:29:23,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163747 2024-11-13T14:29:23,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163747) roll requested 2024-11-13T14:29:23,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163754 2024-11-13T14:29:23,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163754 2024-11-13T14:29:23,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163754) roll requested 2024-11-13T14:29:23,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163760 2024-11-13T14:29:23,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163760 2024-11-13T14:29:23,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163760) roll requested 2024-11-13T14:29:23,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163767 2024-11-13T14:29:23,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163767 2024-11-13T14:29:23,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163767) roll requested 2024-11-13T14:29:23,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163774 2024-11-13T14:29:23,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163774 2024-11-13T14:29:23,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163774) roll requested 2024-11-13T14:29:23,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163781 2024-11-13T14:29:23,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163781 2024-11-13T14:29:23,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163781) roll requested 2024-11-13T14:29:23,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163787 2024-11-13T14:29:23,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163787 2024-11-13T14:29:23,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163787) roll requested 2024-11-13T14:29:23,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163794 2024-11-13T14:29:23,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163794 2024-11-13T14:29:23,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163794) roll requested 2024-11-13T14:29:23,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163800 2024-11-13T14:29:23,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163800 2024-11-13T14:29:23,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163800) roll requested 2024-11-13T14:29:23,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163806 2024-11-13T14:29:23,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163806 2024-11-13T14:29:23,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163806) roll requested 2024-11-13T14:29:23,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163815 2024-11-13T14:29:23,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163815 2024-11-13T14:29:23,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163815) roll requested 2024-11-13T14:29:23,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163821 2024-11-13T14:29:23,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163821 2024-11-13T14:29:23,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163821) roll requested 2024-11-13T14:29:23,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163827 2024-11-13T14:29:23,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163827 2024-11-13T14:29:23,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163827) roll requested 2024-11-13T14:29:23,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163834 2024-11-13T14:29:23,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163834 2024-11-13T14:29:23,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163834) roll requested 2024-11-13T14:29:23,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163841 2024-11-13T14:29:23,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163841 2024-11-13T14:29:23,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163841) roll requested 2024-11-13T14:29:23,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163848 2024-11-13T14:29:23,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163848 2024-11-13T14:29:23,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163848) roll requested 2024-11-13T14:29:23,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163854 2024-11-13T14:29:23,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163854 2024-11-13T14:29:23,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163854) roll requested 2024-11-13T14:29:23,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163860 2024-11-13T14:29:23,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163860 2024-11-13T14:29:23,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163860) roll requested 2024-11-13T14:29:23,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163867 2024-11-13T14:29:23,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163867 2024-11-13T14:29:23,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163867) roll requested 2024-11-13T14:29:23,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163874 2024-11-13T14:29:23,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163874 2024-11-13T14:29:23,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163874) roll requested 2024-11-13T14:29:23,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163881 2024-11-13T14:29:23,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163881 2024-11-13T14:29:23,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163881) roll requested 2024-11-13T14:29:23,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163888 2024-11-13T14:29:23,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163888 2024-11-13T14:29:23,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163888) roll requested 2024-11-13T14:29:23,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163895 2024-11-13T14:29:23,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163895 2024-11-13T14:29:23,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163895) roll requested 2024-11-13T14:29:23,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163903 2024-11-13T14:29:23,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163903 2024-11-13T14:29:23,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163903) roll requested 2024-11-13T14:29:23,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163910 2024-11-13T14:29:23,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163910 2024-11-13T14:29:23,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163910) roll requested 2024-11-13T14:29:23,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163919 2024-11-13T14:29:23,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163919 2024-11-13T14:29:23,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163919) roll requested 2024-11-13T14:29:23,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163925 2024-11-13T14:29:23,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163925 2024-11-13T14:29:23,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163925) roll requested 2024-11-13T14:29:23,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163932 2024-11-13T14:29:23,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163932 2024-11-13T14:29:23,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163932) roll requested 2024-11-13T14:29:23,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163940 2024-11-13T14:29:23,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163940 2024-11-13T14:29:23,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163940) roll requested 2024-11-13T14:29:23,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163948 2024-11-13T14:29:23,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163948 2024-11-13T14:29:23,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163948) roll requested 2024-11-13T14:29:23,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163956 2024-11-13T14:29:23,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163956 2024-11-13T14:29:23,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163956) roll requested 2024-11-13T14:29:23,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163963 2024-11-13T14:29:23,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163963 2024-11-13T14:29:23,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163963) roll requested 2024-11-13T14:29:23,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163972 2024-11-13T14:29:23,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163972 2024-11-13T14:29:23,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163972) roll requested 2024-11-13T14:29:23,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163981 2024-11-13T14:29:23,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163981 2024-11-13T14:29:23,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163981) roll requested 2024-11-13T14:29:23,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163989 2024-11-13T14:29:23,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:23,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163989 2024-11-13T14:29:23,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:23,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:23,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163989) roll requested 2024-11-13T14:29:23,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508163997 2024-11-13T14:29:24,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163997 2024-11-13T14:29:24,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508163997) roll requested 2024-11-13T14:29:24,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164004 2024-11-13T14:29:24,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508163997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164004 2024-11-13T14:29:24,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164004) roll requested 2024-11-13T14:29:24,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164012 2024-11-13T14:29:24,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164012 2024-11-13T14:29:24,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164012) roll requested 2024-11-13T14:29:24,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164018 2024-11-13T14:29:24,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164018 2024-11-13T14:29:24,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164018) roll requested 2024-11-13T14:29:24,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164025 2024-11-13T14:29:24,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164025 2024-11-13T14:29:24,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164025) roll requested 2024-11-13T14:29:24,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164032 2024-11-13T14:29:24,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164032 2024-11-13T14:29:24,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164032) roll requested 2024-11-13T14:29:24,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164042 2024-11-13T14:29:24,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164042 2024-11-13T14:29:24,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164042) roll requested 2024-11-13T14:29:24,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164052 2024-11-13T14:29:24,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164052 2024-11-13T14:29:24,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164052) roll requested 2024-11-13T14:29:24,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164060 2024-11-13T14:29:24,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164060 2024-11-13T14:29:24,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164060) roll requested 2024-11-13T14:29:24,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164067 2024-11-13T14:29:24,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164067 2024-11-13T14:29:24,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164067) roll requested 2024-11-13T14:29:24,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164085 2024-11-13T14:29:24,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164085 2024-11-13T14:29:24,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164085) roll requested 2024-11-13T14:29:24,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164093 2024-11-13T14:29:24,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164093 2024-11-13T14:29:24,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164093) roll requested 2024-11-13T14:29:24,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164103 2024-11-13T14:29:24,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164103 2024-11-13T14:29:24,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164103) roll requested 2024-11-13T14:29:24,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164113 2024-11-13T14:29:24,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164113 2024-11-13T14:29:24,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164113) roll requested 2024-11-13T14:29:24,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164121 2024-11-13T14:29:24,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164121 2024-11-13T14:29:24,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164121) roll requested 2024-11-13T14:29:24,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164128 2024-11-13T14:29:24,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164128 2024-11-13T14:29:24,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164128) roll requested 2024-11-13T14:29:24,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164134 2024-11-13T14:29:24,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164134 2024-11-13T14:29:24,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164134) roll requested 2024-11-13T14:29:24,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164141 2024-11-13T14:29:24,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164141 2024-11-13T14:29:24,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164141) roll requested 2024-11-13T14:29:24,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164151 2024-11-13T14:29:24,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164151 2024-11-13T14:29:24,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164151) roll requested 2024-11-13T14:29:24,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164158 2024-11-13T14:29:24,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164158 2024-11-13T14:29:24,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164158) roll requested 2024-11-13T14:29:24,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164167 2024-11-13T14:29:24,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164167 2024-11-13T14:29:24,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164167) roll requested 2024-11-13T14:29:24,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164174 2024-11-13T14:29:24,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164174 2024-11-13T14:29:24,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164174) roll requested 2024-11-13T14:29:24,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164180 2024-11-13T14:29:24,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164180 2024-11-13T14:29:24,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164180) roll requested 2024-11-13T14:29:24,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164187 2024-11-13T14:29:24,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164187 2024-11-13T14:29:24,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164187) roll requested 2024-11-13T14:29:24,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164194 2024-11-13T14:29:24,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164194 2024-11-13T14:29:24,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164194) roll requested 2024-11-13T14:29:24,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164201 2024-11-13T14:29:24,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164201 2024-11-13T14:29:24,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164201) roll requested 2024-11-13T14:29:24,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164208 2024-11-13T14:29:24,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164208 2024-11-13T14:29:24,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164208) roll requested 2024-11-13T14:29:24,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164215 2024-11-13T14:29:24,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164215 2024-11-13T14:29:24,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164215) roll requested 2024-11-13T14:29:24,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164221 2024-11-13T14:29:24,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164221 2024-11-13T14:29:24,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164221) roll requested 2024-11-13T14:29:24,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164228 2024-11-13T14:29:24,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164228 2024-11-13T14:29:24,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164228) roll requested 2024-11-13T14:29:24,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164237 2024-11-13T14:29:24,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164237 2024-11-13T14:29:24,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164237) roll requested 2024-11-13T14:29:24,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164245 2024-11-13T14:29:24,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164245 2024-11-13T14:29:24,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164245) roll requested 2024-11-13T14:29:24,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164252 2024-11-13T14:29:24,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164252 2024-11-13T14:29:24,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164252) roll requested 2024-11-13T14:29:24,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164259 2024-11-13T14:29:24,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164259 2024-11-13T14:29:24,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164259) roll requested 2024-11-13T14:29:24,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164267 2024-11-13T14:29:24,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164267 2024-11-13T14:29:24,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164267) roll requested 2024-11-13T14:29:24,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164274 2024-11-13T14:29:24,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164274 2024-11-13T14:29:24,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164274) roll requested 2024-11-13T14:29:24,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164282 2024-11-13T14:29:24,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164282 2024-11-13T14:29:24,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164282) roll requested 2024-11-13T14:29:24,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164289 2024-11-13T14:29:24,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164289 2024-11-13T14:29:24,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164289) roll requested 2024-11-13T14:29:24,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164298 2024-11-13T14:29:24,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164298 2024-11-13T14:29:24,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164298) roll requested 2024-11-13T14:29:24,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164306 2024-11-13T14:29:24,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164306 2024-11-13T14:29:24,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164306) roll requested 2024-11-13T14:29:24,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164312 2024-11-13T14:29:24,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164312 2024-11-13T14:29:24,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164312) roll requested 2024-11-13T14:29:24,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164318 2024-11-13T14:29:24,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164318 2024-11-13T14:29:24,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164318) roll requested 2024-11-13T14:29:24,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164324 2024-11-13T14:29:24,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164324 2024-11-13T14:29:24,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164324) roll requested 2024-11-13T14:29:24,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164330 2024-11-13T14:29:24,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164330 2024-11-13T14:29:24,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164330) roll requested 2024-11-13T14:29:24,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164340 2024-11-13T14:29:24,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164340 2024-11-13T14:29:24,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164340) roll requested 2024-11-13T14:29:24,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164346 2024-11-13T14:29:24,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164346 2024-11-13T14:29:24,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164346) roll requested 2024-11-13T14:29:24,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164355 2024-11-13T14:29:24,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164355 2024-11-13T14:29:24,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164355) roll requested 2024-11-13T14:29:24,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164361 2024-11-13T14:29:24,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164361 2024-11-13T14:29:24,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164361) roll requested 2024-11-13T14:29:24,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164366 2024-11-13T14:29:24,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164366 2024-11-13T14:29:24,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164366) roll requested 2024-11-13T14:29:24,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164375 2024-11-13T14:29:24,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164375 2024-11-13T14:29:24,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164375) roll requested 2024-11-13T14:29:24,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164385 2024-11-13T14:29:24,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164385 2024-11-13T14:29:24,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164385) roll requested 2024-11-13T14:29:24,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164391 2024-11-13T14:29:24,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164391 2024-11-13T14:29:24,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164391) roll requested 2024-11-13T14:29:24,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164397 2024-11-13T14:29:24,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164397 2024-11-13T14:29:24,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164397) roll requested 2024-11-13T14:29:24,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164405 2024-11-13T14:29:24,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164405 2024-11-13T14:29:24,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164405) roll requested 2024-11-13T14:29:24,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164415 2024-11-13T14:29:24,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164415 2024-11-13T14:29:24,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164415) roll requested 2024-11-13T14:29:24,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164423 2024-11-13T14:29:24,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164423 2024-11-13T14:29:24,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164423) roll requested 2024-11-13T14:29:24,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164430 2024-11-13T14:29:24,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164430 2024-11-13T14:29:24,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164430) roll requested 2024-11-13T14:29:24,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164437 2024-11-13T14:29:24,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164437 2024-11-13T14:29:24,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164437) roll requested 2024-11-13T14:29:24,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164446 2024-11-13T14:29:24,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164446 2024-11-13T14:29:24,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164446) roll requested 2024-11-13T14:29:24,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164458 2024-11-13T14:29:24,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164458 2024-11-13T14:29:24,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164458) roll requested 2024-11-13T14:29:24,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164465 2024-11-13T14:29:24,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164465 2024-11-13T14:29:24,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164465) roll requested 2024-11-13T14:29:24,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164474 2024-11-13T14:29:24,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164474 2024-11-13T14:29:24,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164474) roll requested 2024-11-13T14:29:24,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164481 2024-11-13T14:29:24,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164481 2024-11-13T14:29:24,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164481) roll requested 2024-11-13T14:29:24,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164490 2024-11-13T14:29:24,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164490 2024-11-13T14:29:24,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164490) roll requested 2024-11-13T14:29:24,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164497 2024-11-13T14:29:24,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164497 2024-11-13T14:29:24,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164497) roll requested 2024-11-13T14:29:24,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164505 2024-11-13T14:29:24,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164505 2024-11-13T14:29:24,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164505) roll requested 2024-11-13T14:29:24,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164511 2024-11-13T14:29:24,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164511 2024-11-13T14:29:24,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164511) roll requested 2024-11-13T14:29:24,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164518 2024-11-13T14:29:24,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164518 2024-11-13T14:29:24,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164518) roll requested 2024-11-13T14:29:24,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164525 2024-11-13T14:29:24,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164525 2024-11-13T14:29:24,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164525) roll requested 2024-11-13T14:29:24,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164531 2024-11-13T14:29:24,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164531 2024-11-13T14:29:24,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164531) roll requested 2024-11-13T14:29:24,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164537 2024-11-13T14:29:24,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164537 2024-11-13T14:29:24,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164537) roll requested 2024-11-13T14:29:24,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164544 2024-11-13T14:29:24,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164544 2024-11-13T14:29:24,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164544) roll requested 2024-11-13T14:29:24,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164552 2024-11-13T14:29:24,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164552 2024-11-13T14:29:24,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164552) roll requested 2024-11-13T14:29:24,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164558 2024-11-13T14:29:24,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164558 2024-11-13T14:29:24,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164558) roll requested 2024-11-13T14:29:24,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164564 2024-11-13T14:29:24,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164564 2024-11-13T14:29:24,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164564) roll requested 2024-11-13T14:29:24,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164572 2024-11-13T14:29:24,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164572 2024-11-13T14:29:24,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164572) roll requested 2024-11-13T14:29:24,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164579 2024-11-13T14:29:24,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164579 2024-11-13T14:29:24,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164579) roll requested 2024-11-13T14:29:24,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164590 2024-11-13T14:29:24,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164590 2024-11-13T14:29:24,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164590) roll requested 2024-11-13T14:29:24,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164603 2024-11-13T14:29:24,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164603 2024-11-13T14:29:24,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164603) roll requested 2024-11-13T14:29:24,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164609 2024-11-13T14:29:24,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164609 2024-11-13T14:29:24,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164609) roll requested 2024-11-13T14:29:24,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164615 2024-11-13T14:29:24,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164615 2024-11-13T14:29:24,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164615) roll requested 2024-11-13T14:29:24,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164625 2024-11-13T14:29:24,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164625 2024-11-13T14:29:24,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164625) roll requested 2024-11-13T14:29:24,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164633 2024-11-13T14:29:24,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164633 2024-11-13T14:29:24,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164633) roll requested 2024-11-13T14:29:24,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164640 2024-11-13T14:29:24,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164640 2024-11-13T14:29:24,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164640) roll requested 2024-11-13T14:29:24,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164648 2024-11-13T14:29:24,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164648 2024-11-13T14:29:24,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164648) roll requested 2024-11-13T14:29:24,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164656 2024-11-13T14:29:24,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164656 2024-11-13T14:29:24,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164656) roll requested 2024-11-13T14:29:24,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164663 2024-11-13T14:29:24,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164663 2024-11-13T14:29:24,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164663) roll requested 2024-11-13T14:29:24,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164672 2024-11-13T14:29:24,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164672 2024-11-13T14:29:24,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164672) roll requested 2024-11-13T14:29:24,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164679 2024-11-13T14:29:24,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164679 2024-11-13T14:29:24,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164679) roll requested 2024-11-13T14:29:24,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164687 2024-11-13T14:29:24,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164687 2024-11-13T14:29:24,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164687) roll requested 2024-11-13T14:29:24,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164695 2024-11-13T14:29:24,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164695 2024-11-13T14:29:24,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164695) roll requested 2024-11-13T14:29:24,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164704 2024-11-13T14:29:24,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164704 2024-11-13T14:29:24,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164704) roll requested 2024-11-13T14:29:24,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164712 2024-11-13T14:29:24,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164712 2024-11-13T14:29:24,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164712) roll requested 2024-11-13T14:29:24,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164719 2024-11-13T14:29:24,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164719 2024-11-13T14:29:24,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164719) roll requested 2024-11-13T14:29:24,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164725 2024-11-13T14:29:24,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164725 2024-11-13T14:29:24,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164725) roll requested 2024-11-13T14:29:24,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164731 2024-11-13T14:29:24,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164731 2024-11-13T14:29:24,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164731) roll requested 2024-11-13T14:29:24,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164745 2024-11-13T14:29:24,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164745 2024-11-13T14:29:24,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164745) roll requested 2024-11-13T14:29:24,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164753 2024-11-13T14:29:24,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164753 2024-11-13T14:29:24,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164753) roll requested 2024-11-13T14:29:24,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164759 2024-11-13T14:29:24,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164759 2024-11-13T14:29:24,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164759) roll requested 2024-11-13T14:29:24,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164765 2024-11-13T14:29:24,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164765 2024-11-13T14:29:24,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164765) roll requested 2024-11-13T14:29:24,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164772 2024-11-13T14:29:24,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164772 2024-11-13T14:29:24,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164772) roll requested 2024-11-13T14:29:24,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164779 2024-11-13T14:29:24,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164779 2024-11-13T14:29:24,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164779) roll requested 2024-11-13T14:29:24,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164790 2024-11-13T14:29:24,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164790 2024-11-13T14:29:24,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164790) roll requested 2024-11-13T14:29:24,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164796 2024-11-13T14:29:24,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164796 2024-11-13T14:29:24,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164796) roll requested 2024-11-13T14:29:24,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164803 2024-11-13T14:29:24,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164803 2024-11-13T14:29:24,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164803) roll requested 2024-11-13T14:29:24,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164810 2024-11-13T14:29:24,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164810 2024-11-13T14:29:24,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164810) roll requested 2024-11-13T14:29:24,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164820 2024-11-13T14:29:24,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164820 2024-11-13T14:29:24,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164820) roll requested 2024-11-13T14:29:24,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164827 2024-11-13T14:29:24,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164827 2024-11-13T14:29:24,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164827) roll requested 2024-11-13T14:29:24,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164837 2024-11-13T14:29:24,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164837 2024-11-13T14:29:24,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164837) roll requested 2024-11-13T14:29:24,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164843 2024-11-13T14:29:24,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164843 2024-11-13T14:29:24,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164843) roll requested 2024-11-13T14:29:24,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164854 2024-11-13T14:29:24,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164854 2024-11-13T14:29:24,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164854) roll requested 2024-11-13T14:29:24,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164862 2024-11-13T14:29:24,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164862 2024-11-13T14:29:24,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164862) roll requested 2024-11-13T14:29:24,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164869 2024-11-13T14:29:24,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164869 2024-11-13T14:29:24,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164869) roll requested 2024-11-13T14:29:24,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164877 2024-11-13T14:29:24,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164877 2024-11-13T14:29:24,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164877) roll requested 2024-11-13T14:29:24,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164884 2024-11-13T14:29:24,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164884 2024-11-13T14:29:24,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164884) roll requested 2024-11-13T14:29:24,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164892 2024-11-13T14:29:24,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164892 2024-11-13T14:29:24,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164892) roll requested 2024-11-13T14:29:24,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164901 2024-11-13T14:29:24,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164901 2024-11-13T14:29:24,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164901) roll requested 2024-11-13T14:29:24,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164908 2024-11-13T14:29:24,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164908 2024-11-13T14:29:24,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164908) roll requested 2024-11-13T14:29:24,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164915 2024-11-13T14:29:24,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164915 2024-11-13T14:29:24,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164915) roll requested 2024-11-13T14:29:24,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164923 2024-11-13T14:29:24,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164923 2024-11-13T14:29:24,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164923) roll requested 2024-11-13T14:29:24,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164932 2024-11-13T14:29:24,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164932 2024-11-13T14:29:24,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164932) roll requested 2024-11-13T14:29:24,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164939 2024-11-13T14:29:24,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164939 2024-11-13T14:29:24,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164939) roll requested 2024-11-13T14:29:24,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164945 2024-11-13T14:29:24,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164945 2024-11-13T14:29:24,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164945) roll requested 2024-11-13T14:29:24,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164952 2024-11-13T14:29:24,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164952 2024-11-13T14:29:24,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164952) roll requested 2024-11-13T14:29:24,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164958 2024-11-13T14:29:24,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164958 2024-11-13T14:29:24,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164958) roll requested 2024-11-13T14:29:24,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164967 2024-11-13T14:29:24,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164967 2024-11-13T14:29:24,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164967) roll requested 2024-11-13T14:29:24,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164977 2024-11-13T14:29:24,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164977 2024-11-13T14:29:24,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164977) roll requested 2024-11-13T14:29:24,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164984 2024-11-13T14:29:24,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:24,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164984 2024-11-13T14:29:24,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:24,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:24,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164984) roll requested 2024-11-13T14:29:24,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508164994 2024-11-13T14:29:25,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164994 2024-11-13T14:29:25,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508164994) roll requested 2024-11-13T14:29:25,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165002 2024-11-13T14:29:25,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508164994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165002 2024-11-13T14:29:25,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165002) roll requested 2024-11-13T14:29:25,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165009 2024-11-13T14:29:25,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165009 2024-11-13T14:29:25,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165009) roll requested 2024-11-13T14:29:25,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165022 2024-11-13T14:29:25,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165022 2024-11-13T14:29:25,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165022) roll requested 2024-11-13T14:29:25,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165028 2024-11-13T14:29:25,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165028 2024-11-13T14:29:25,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165028) roll requested 2024-11-13T14:29:25,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165036 2024-11-13T14:29:25,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165036 2024-11-13T14:29:25,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165036) roll requested 2024-11-13T14:29:25,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165044 2024-11-13T14:29:25,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165044 2024-11-13T14:29:25,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165044) roll requested 2024-11-13T14:29:25,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165051 2024-11-13T14:29:25,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165051 2024-11-13T14:29:25,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165051) roll requested 2024-11-13T14:29:25,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165057 2024-11-13T14:29:25,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165057 2024-11-13T14:29:25,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165057) roll requested 2024-11-13T14:29:25,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165064 2024-11-13T14:29:25,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165064 2024-11-13T14:29:25,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165064) roll requested 2024-11-13T14:29:25,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165072 2024-11-13T14:29:25,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165072 2024-11-13T14:29:25,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165072) roll requested 2024-11-13T14:29:25,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165080 2024-11-13T14:29:25,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165080 2024-11-13T14:29:25,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165080) roll requested 2024-11-13T14:29:25,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165088 2024-11-13T14:29:25,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165088 2024-11-13T14:29:25,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165088) roll requested 2024-11-13T14:29:25,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165099 2024-11-13T14:29:25,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165099 2024-11-13T14:29:25,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165099) roll requested 2024-11-13T14:29:25,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165105 2024-11-13T14:29:25,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165105 2024-11-13T14:29:25,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165105) roll requested 2024-11-13T14:29:25,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165112 2024-11-13T14:29:25,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165112 2024-11-13T14:29:25,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165112) roll requested 2024-11-13T14:29:25,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165120 2024-11-13T14:29:25,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165120 2024-11-13T14:29:25,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165120) roll requested 2024-11-13T14:29:25,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165129 2024-11-13T14:29:25,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165129 2024-11-13T14:29:25,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165129) roll requested 2024-11-13T14:29:25,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165138 2024-11-13T14:29:25,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165138 2024-11-13T14:29:25,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165138) roll requested 2024-11-13T14:29:25,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165148 2024-11-13T14:29:25,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165148 2024-11-13T14:29:25,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165148) roll requested 2024-11-13T14:29:25,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165159 2024-11-13T14:29:25,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165159 2024-11-13T14:29:25,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165159) roll requested 2024-11-13T14:29:25,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165166 2024-11-13T14:29:25,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165166 2024-11-13T14:29:25,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165166) roll requested 2024-11-13T14:29:25,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165172 2024-11-13T14:29:25,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165172 2024-11-13T14:29:25,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165172) roll requested 2024-11-13T14:29:25,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165178 2024-11-13T14:29:25,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165178 2024-11-13T14:29:25,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165178) roll requested 2024-11-13T14:29:25,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165185 2024-11-13T14:29:25,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165185 2024-11-13T14:29:25,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165185) roll requested 2024-11-13T14:29:25,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165192 2024-11-13T14:29:25,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165192 2024-11-13T14:29:25,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165192) roll requested 2024-11-13T14:29:25,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165199 2024-11-13T14:29:25,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165199 2024-11-13T14:29:25,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165199) roll requested 2024-11-13T14:29:25,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165211 2024-11-13T14:29:25,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165211 2024-11-13T14:29:25,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165211) roll requested 2024-11-13T14:29:25,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165217 2024-11-13T14:29:25,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165217 2024-11-13T14:29:25,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165217) roll requested 2024-11-13T14:29:25,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165223 2024-11-13T14:29:25,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165223 2024-11-13T14:29:25,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165223) roll requested 2024-11-13T14:29:25,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165229 2024-11-13T14:29:25,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165229 2024-11-13T14:29:25,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165229) roll requested 2024-11-13T14:29:25,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165235 2024-11-13T14:29:25,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165235 2024-11-13T14:29:25,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165235) roll requested 2024-11-13T14:29:25,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165244 2024-11-13T14:29:25,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165244 2024-11-13T14:29:25,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165244) roll requested 2024-11-13T14:29:25,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165250 2024-11-13T14:29:25,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165250 2024-11-13T14:29:25,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165250) roll requested 2024-11-13T14:29:25,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165258 2024-11-13T14:29:25,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165258 2024-11-13T14:29:25,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165258) roll requested 2024-11-13T14:29:25,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165264 2024-11-13T14:29:25,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165264 2024-11-13T14:29:25,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165264) roll requested 2024-11-13T14:29:25,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165272 2024-11-13T14:29:25,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165272 2024-11-13T14:29:25,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165272) roll requested 2024-11-13T14:29:25,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165280 2024-11-13T14:29:25,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165280 2024-11-13T14:29:25,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165280) roll requested 2024-11-13T14:29:25,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165289 2024-11-13T14:29:25,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165289 2024-11-13T14:29:25,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165289) roll requested 2024-11-13T14:29:25,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165298 2024-11-13T14:29:25,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165298 2024-11-13T14:29:25,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165298) roll requested 2024-11-13T14:29:25,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165309 2024-11-13T14:29:25,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165309 2024-11-13T14:29:25,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165309) roll requested 2024-11-13T14:29:25,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165317 2024-11-13T14:29:25,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165317 2024-11-13T14:29:25,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165317) roll requested 2024-11-13T14:29:25,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165325 2024-11-13T14:29:25,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165325 2024-11-13T14:29:25,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165325) roll requested 2024-11-13T14:29:25,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165334 2024-11-13T14:29:25,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165334 2024-11-13T14:29:25,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165334) roll requested 2024-11-13T14:29:25,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165343 2024-11-13T14:29:25,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165343 2024-11-13T14:29:25,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165343) roll requested 2024-11-13T14:29:25,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165351 2024-11-13T14:29:25,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165351 2024-11-13T14:29:25,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165351) roll requested 2024-11-13T14:29:25,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165360 2024-11-13T14:29:25,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165360 2024-11-13T14:29:25,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165360) roll requested 2024-11-13T14:29:25,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165369 2024-11-13T14:29:25,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165369 2024-11-13T14:29:25,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165369) roll requested 2024-11-13T14:29:25,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165377 2024-11-13T14:29:25,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165377 2024-11-13T14:29:25,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165377) roll requested 2024-11-13T14:29:25,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165385 2024-11-13T14:29:25,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165385 2024-11-13T14:29:25,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165385) roll requested 2024-11-13T14:29:25,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165400 2024-11-13T14:29:25,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165400 2024-11-13T14:29:25,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165400) roll requested 2024-11-13T14:29:25,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165408 2024-11-13T14:29:25,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165408 2024-11-13T14:29:25,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165408) roll requested 2024-11-13T14:29:25,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165413 2024-11-13T14:29:25,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165413 2024-11-13T14:29:25,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165413) roll requested 2024-11-13T14:29:25,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165419 2024-11-13T14:29:25,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165419 2024-11-13T14:29:25,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165419) roll requested 2024-11-13T14:29:25,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165426 2024-11-13T14:29:25,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165426 2024-11-13T14:29:25,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165426) roll requested 2024-11-13T14:29:25,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165434 2024-11-13T14:29:25,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165434 2024-11-13T14:29:25,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165434) roll requested 2024-11-13T14:29:25,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165445 2024-11-13T14:29:25,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165445 2024-11-13T14:29:25,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165445) roll requested 2024-11-13T14:29:25,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165455 2024-11-13T14:29:25,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165455 2024-11-13T14:29:25,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165455) roll requested 2024-11-13T14:29:25,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165467 2024-11-13T14:29:25,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165467 2024-11-13T14:29:25,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165467) roll requested 2024-11-13T14:29:25,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165474 2024-11-13T14:29:25,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165474 2024-11-13T14:29:25,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165474) roll requested 2024-11-13T14:29:25,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165483 2024-11-13T14:29:25,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165483 2024-11-13T14:29:25,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165483) roll requested 2024-11-13T14:29:25,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165491 2024-11-13T14:29:25,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165491 2024-11-13T14:29:25,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165491) roll requested 2024-11-13T14:29:25,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165500 2024-11-13T14:29:25,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165500 2024-11-13T14:29:25,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165500) roll requested 2024-11-13T14:29:25,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165506 2024-11-13T14:29:25,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165506 2024-11-13T14:29:25,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165506) roll requested 2024-11-13T14:29:25,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165515 2024-11-13T14:29:25,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165515 2024-11-13T14:29:25,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165515) roll requested 2024-11-13T14:29:25,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165522 2024-11-13T14:29:25,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165522 2024-11-13T14:29:25,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165522) roll requested 2024-11-13T14:29:25,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165530 2024-11-13T14:29:25,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165530 2024-11-13T14:29:25,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165530) roll requested 2024-11-13T14:29:25,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165539 2024-11-13T14:29:25,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165539 2024-11-13T14:29:25,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165539) roll requested 2024-11-13T14:29:25,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165549 2024-11-13T14:29:25,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165549 2024-11-13T14:29:25,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165549) roll requested 2024-11-13T14:29:25,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165560 2024-11-13T14:29:25,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165560 2024-11-13T14:29:25,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165560) roll requested 2024-11-13T14:29:25,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165565 2024-11-13T14:29:25,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165565 2024-11-13T14:29:25,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165565) roll requested 2024-11-13T14:29:25,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165572 2024-11-13T14:29:25,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165572 2024-11-13T14:29:25,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165572) roll requested 2024-11-13T14:29:25,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165580 2024-11-13T14:29:25,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165580 2024-11-13T14:29:25,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165580) roll requested 2024-11-13T14:29:25,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165589 2024-11-13T14:29:25,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165589 2024-11-13T14:29:25,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165589) roll requested 2024-11-13T14:29:25,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165599 2024-11-13T14:29:25,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165599 2024-11-13T14:29:25,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165599) roll requested 2024-11-13T14:29:25,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165608 2024-11-13T14:29:25,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165608 2024-11-13T14:29:25,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165608) roll requested 2024-11-13T14:29:25,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165617 2024-11-13T14:29:25,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165617 2024-11-13T14:29:25,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165617) roll requested 2024-11-13T14:29:25,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165626 2024-11-13T14:29:25,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165626 2024-11-13T14:29:25,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165626) roll requested 2024-11-13T14:29:25,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165634 2024-11-13T14:29:25,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165634 2024-11-13T14:29:25,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165634) roll requested 2024-11-13T14:29:25,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165642 2024-11-13T14:29:25,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165642 2024-11-13T14:29:25,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165642) roll requested 2024-11-13T14:29:25,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165651 2024-11-13T14:29:25,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165651 2024-11-13T14:29:25,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165651) roll requested 2024-11-13T14:29:25,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165661 2024-11-13T14:29:25,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165661 2024-11-13T14:29:25,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165661) roll requested 2024-11-13T14:29:25,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165671 2024-11-13T14:29:25,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165671 2024-11-13T14:29:25,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165671) roll requested 2024-11-13T14:29:25,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165742 2024-11-13T14:29:25,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165742 2024-11-13T14:29:25,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165742) roll requested 2024-11-13T14:29:25,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165753 2024-11-13T14:29:25,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165753 2024-11-13T14:29:25,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165753) roll requested 2024-11-13T14:29:25,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165760 2024-11-13T14:29:25,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165760 2024-11-13T14:29:25,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165760) roll requested 2024-11-13T14:29:25,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165770 2024-11-13T14:29:25,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165770 2024-11-13T14:29:25,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165770) roll requested 2024-11-13T14:29:25,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165779 2024-11-13T14:29:25,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165779 2024-11-13T14:29:25,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165779) roll requested 2024-11-13T14:29:25,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165789 2024-11-13T14:29:25,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165789 2024-11-13T14:29:25,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165789) roll requested 2024-11-13T14:29:25,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165798 2024-11-13T14:29:25,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165798 2024-11-13T14:29:25,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165798) roll requested 2024-11-13T14:29:25,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165804 2024-11-13T14:29:25,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165804 2024-11-13T14:29:25,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165804) roll requested 2024-11-13T14:29:25,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165813 2024-11-13T14:29:25,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165813 2024-11-13T14:29:25,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165813) roll requested 2024-11-13T14:29:25,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165820 2024-11-13T14:29:25,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165820 2024-11-13T14:29:25,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165820) roll requested 2024-11-13T14:29:25,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165828 2024-11-13T14:29:25,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165828 2024-11-13T14:29:25,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165828) roll requested 2024-11-13T14:29:25,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165836 2024-11-13T14:29:25,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165836 2024-11-13T14:29:25,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165836) roll requested 2024-11-13T14:29:25,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165845 2024-11-13T14:29:25,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165845 2024-11-13T14:29:25,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165845) roll requested 2024-11-13T14:29:25,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165854 2024-11-13T14:29:25,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165854 2024-11-13T14:29:25,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165854) roll requested 2024-11-13T14:29:25,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165860 2024-11-13T14:29:25,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165860 2024-11-13T14:29:25,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165860) roll requested 2024-11-13T14:29:25,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165869 2024-11-13T14:29:25,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165869 2024-11-13T14:29:25,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165869) roll requested 2024-11-13T14:29:25,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165876 2024-11-13T14:29:25,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165876 2024-11-13T14:29:25,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165876) roll requested 2024-11-13T14:29:25,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165883 2024-11-13T14:29:25,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165883 2024-11-13T14:29:25,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165883) roll requested 2024-11-13T14:29:25,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165891 2024-11-13T14:29:25,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165891 2024-11-13T14:29:25,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165891) roll requested 2024-11-13T14:29:25,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165900 2024-11-13T14:29:25,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165900 2024-11-13T14:29:25,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165900) roll requested 2024-11-13T14:29:25,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165910 2024-11-13T14:29:25,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165910 2024-11-13T14:29:25,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165910) roll requested 2024-11-13T14:29:25,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165919 2024-11-13T14:29:25,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165919 2024-11-13T14:29:25,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165919) roll requested 2024-11-13T14:29:25,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165925 2024-11-13T14:29:25,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165925 2024-11-13T14:29:25,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165925) roll requested 2024-11-13T14:29:25,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165932 2024-11-13T14:29:25,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165932 2024-11-13T14:29:25,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165932) roll requested 2024-11-13T14:29:25,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165939 2024-11-13T14:29:25,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165939 2024-11-13T14:29:25,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165939) roll requested 2024-11-13T14:29:25,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165945 2024-11-13T14:29:25,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165945 2024-11-13T14:29:25,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165945) roll requested 2024-11-13T14:29:25,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165954 2024-11-13T14:29:25,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165954 2024-11-13T14:29:25,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165954) roll requested 2024-11-13T14:29:25,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165960 2024-11-13T14:29:25,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165960 2024-11-13T14:29:25,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165960) roll requested 2024-11-13T14:29:25,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165966 2024-11-13T14:29:25,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165966 2024-11-13T14:29:25,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165966) roll requested 2024-11-13T14:29:25,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165974 2024-11-13T14:29:25,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165974 2024-11-13T14:29:25,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165974) roll requested 2024-11-13T14:29:25,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165982 2024-11-13T14:29:25,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165982 2024-11-13T14:29:25,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165982) roll requested 2024-11-13T14:29:25,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165989 2024-11-13T14:29:25,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:25,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165989 2024-11-13T14:29:25,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:25,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:25,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165989) roll requested 2024-11-13T14:29:25,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508165996 2024-11-13T14:29:26,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165996 2024-11-13T14:29:26,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508165996) roll requested 2024-11-13T14:29:26,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166004 2024-11-13T14:29:26,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508165996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166004 2024-11-13T14:29:26,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166004) roll requested 2024-11-13T14:29:26,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166013 2024-11-13T14:29:26,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166013 2024-11-13T14:29:26,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166013) roll requested 2024-11-13T14:29:26,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166024 2024-11-13T14:29:26,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166024 2024-11-13T14:29:26,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166024) roll requested 2024-11-13T14:29:26,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166032 2024-11-13T14:29:26,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166032 2024-11-13T14:29:26,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166032) roll requested 2024-11-13T14:29:26,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166040 2024-11-13T14:29:26,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166040 2024-11-13T14:29:26,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166040) roll requested 2024-11-13T14:29:26,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166050 2024-11-13T14:29:26,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166050 2024-11-13T14:29:26,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166050) roll requested 2024-11-13T14:29:26,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166056 2024-11-13T14:29:26,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166056 2024-11-13T14:29:26,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166056) roll requested 2024-11-13T14:29:26,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166064 2024-11-13T14:29:26,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166064 2024-11-13T14:29:26,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166064) roll requested 2024-11-13T14:29:26,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166072 2024-11-13T14:29:26,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166072 2024-11-13T14:29:26,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166072) roll requested 2024-11-13T14:29:26,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166083 2024-11-13T14:29:26,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166083 2024-11-13T14:29:26,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166083) roll requested 2024-11-13T14:29:26,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166092 2024-11-13T14:29:26,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166092 2024-11-13T14:29:26,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166092) roll requested 2024-11-13T14:29:26,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166099 2024-11-13T14:29:26,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166099 2024-11-13T14:29:26,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166099) roll requested 2024-11-13T14:29:26,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166105 2024-11-13T14:29:26,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166105 2024-11-13T14:29:26,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166105) roll requested 2024-11-13T14:29:26,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166111 2024-11-13T14:29:26,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166111 2024-11-13T14:29:26,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166111) roll requested 2024-11-13T14:29:26,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166118 2024-11-13T14:29:26,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166118 2024-11-13T14:29:26,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166118) roll requested 2024-11-13T14:29:26,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166124 2024-11-13T14:29:26,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166124 2024-11-13T14:29:26,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166124) roll requested 2024-11-13T14:29:26,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166134 2024-11-13T14:29:26,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166134 2024-11-13T14:29:26,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166134) roll requested 2024-11-13T14:29:26,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166141 2024-11-13T14:29:26,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166141 2024-11-13T14:29:26,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166141) roll requested 2024-11-13T14:29:26,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166147 2024-11-13T14:29:26,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166147 2024-11-13T14:29:26,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166147) roll requested 2024-11-13T14:29:26,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166153 2024-11-13T14:29:26,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166153 2024-11-13T14:29:26,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166153) roll requested 2024-11-13T14:29:26,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166160 2024-11-13T14:29:26,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166160 2024-11-13T14:29:26,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166160) roll requested 2024-11-13T14:29:26,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166168 2024-11-13T14:29:26,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166168 2024-11-13T14:29:26,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166168) roll requested 2024-11-13T14:29:26,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166177 2024-11-13T14:29:26,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166177 2024-11-13T14:29:26,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166177) roll requested 2024-11-13T14:29:26,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166186 2024-11-13T14:29:26,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166186 2024-11-13T14:29:26,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166186) roll requested 2024-11-13T14:29:26,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166197 2024-11-13T14:29:26,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166197 2024-11-13T14:29:26,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166197) roll requested 2024-11-13T14:29:26,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166206 2024-11-13T14:29:26,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166206 2024-11-13T14:29:26,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166206) roll requested 2024-11-13T14:29:26,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166217 2024-11-13T14:29:26,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166217 2024-11-13T14:29:26,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166217) roll requested 2024-11-13T14:29:26,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166228 2024-11-13T14:29:26,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166228 2024-11-13T14:29:26,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166228) roll requested 2024-11-13T14:29:26,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166234 2024-11-13T14:29:26,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166234 2024-11-13T14:29:26,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166234) roll requested 2024-11-13T14:29:26,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166242 2024-11-13T14:29:26,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166242 2024-11-13T14:29:26,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166242) roll requested 2024-11-13T14:29:26,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166248 2024-11-13T14:29:26,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166248 2024-11-13T14:29:26,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166248) roll requested 2024-11-13T14:29:26,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166253 2024-11-13T14:29:26,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166253 2024-11-13T14:29:26,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166253) roll requested 2024-11-13T14:29:26,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166263 2024-11-13T14:29:26,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166263 2024-11-13T14:29:26,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166263) roll requested 2024-11-13T14:29:26,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166268 2024-11-13T14:29:26,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166268 2024-11-13T14:29:26,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166268) roll requested 2024-11-13T14:29:26,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166275 2024-11-13T14:29:26,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166275 2024-11-13T14:29:26,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166275) roll requested 2024-11-13T14:29:26,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166284 2024-11-13T14:29:26,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166284 2024-11-13T14:29:26,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166284) roll requested 2024-11-13T14:29:26,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166291 2024-11-13T14:29:26,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166291 2024-11-13T14:29:26,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166291) roll requested 2024-11-13T14:29:26,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166300 2024-11-13T14:29:26,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166300 2024-11-13T14:29:26,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166300) roll requested 2024-11-13T14:29:26,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166306 2024-11-13T14:29:26,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166306 2024-11-13T14:29:26,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166306) roll requested 2024-11-13T14:29:26,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166312 2024-11-13T14:29:26,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166312 2024-11-13T14:29:26,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166312) roll requested 2024-11-13T14:29:26,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166320 2024-11-13T14:29:26,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166320 2024-11-13T14:29:26,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166320) roll requested 2024-11-13T14:29:26,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166326 2024-11-13T14:29:26,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166326 2024-11-13T14:29:26,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166326) roll requested 2024-11-13T14:29:26,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166332 2024-11-13T14:29:26,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166332 2024-11-13T14:29:26,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166332) roll requested 2024-11-13T14:29:26,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166338 2024-11-13T14:29:26,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166338 2024-11-13T14:29:26,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166338) roll requested 2024-11-13T14:29:26,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166392 2024-11-13T14:29:26,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166392 2024-11-13T14:29:26,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166392) roll requested 2024-11-13T14:29:26,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166404 2024-11-13T14:29:26,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166404 2024-11-13T14:29:26,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166404) roll requested 2024-11-13T14:29:26,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166409 2024-11-13T14:29:26,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166409 2024-11-13T14:29:26,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166409) roll requested 2024-11-13T14:29:26,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166414 2024-11-13T14:29:26,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166414 2024-11-13T14:29:26,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166414) roll requested 2024-11-13T14:29:26,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166422 2024-11-13T14:29:26,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166422 2024-11-13T14:29:26,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166422) roll requested 2024-11-13T14:29:26,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166432 2024-11-13T14:29:26,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166432 2024-11-13T14:29:26,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166432) roll requested 2024-11-13T14:29:26,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166440 2024-11-13T14:29:26,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166440 2024-11-13T14:29:26,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166440) roll requested 2024-11-13T14:29:26,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166447 2024-11-13T14:29:26,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166447 2024-11-13T14:29:26,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166447) roll requested 2024-11-13T14:29:26,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166454 2024-11-13T14:29:26,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166454 2024-11-13T14:29:26,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166454) roll requested 2024-11-13T14:29:26,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166460 2024-11-13T14:29:26,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166460 2024-11-13T14:29:26,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166460) roll requested 2024-11-13T14:29:26,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166466 2024-11-13T14:29:26,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166466 2024-11-13T14:29:26,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166466) roll requested 2024-11-13T14:29:26,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166471 2024-11-13T14:29:26,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166471 2024-11-13T14:29:26,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166471) roll requested 2024-11-13T14:29:26,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166476 2024-11-13T14:29:26,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166476 2024-11-13T14:29:26,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166476) roll requested 2024-11-13T14:29:26,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166483 2024-11-13T14:29:26,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166483 2024-11-13T14:29:26,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166483) roll requested 2024-11-13T14:29:26,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166490 2024-11-13T14:29:26,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166490 2024-11-13T14:29:26,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166490) roll requested 2024-11-13T14:29:26,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166499 2024-11-13T14:29:26,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166499 2024-11-13T14:29:26,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166499) roll requested 2024-11-13T14:29:26,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166508 2024-11-13T14:29:26,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166508 2024-11-13T14:29:26,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166508) roll requested 2024-11-13T14:29:26,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166517 2024-11-13T14:29:26,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166517 2024-11-13T14:29:26,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166517) roll requested 2024-11-13T14:29:26,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166524 2024-11-13T14:29:26,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166524 2024-11-13T14:29:26,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166524) roll requested 2024-11-13T14:29:26,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166532 2024-11-13T14:29:26,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166532 2024-11-13T14:29:26,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166532) roll requested 2024-11-13T14:29:26,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166538 2024-11-13T14:29:26,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166538 2024-11-13T14:29:26,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166538) roll requested 2024-11-13T14:29:26,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166549 2024-11-13T14:29:26,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166549 2024-11-13T14:29:26,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166549) roll requested 2024-11-13T14:29:26,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166556 2024-11-13T14:29:26,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166556 2024-11-13T14:29:26,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166556) roll requested 2024-11-13T14:29:26,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166561 2024-11-13T14:29:26,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166561 2024-11-13T14:29:26,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166561) roll requested 2024-11-13T14:29:26,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166569 2024-11-13T14:29:26,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166569 2024-11-13T14:29:26,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166569) roll requested 2024-11-13T14:29:26,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166575 2024-11-13T14:29:26,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166575 2024-11-13T14:29:26,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166575) roll requested 2024-11-13T14:29:26,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166580 2024-11-13T14:29:26,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166580 2024-11-13T14:29:26,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166580) roll requested 2024-11-13T14:29:26,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166586 2024-11-13T14:29:26,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166586 2024-11-13T14:29:26,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166586) roll requested 2024-11-13T14:29:26,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166592 2024-11-13T14:29:26,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166592 2024-11-13T14:29:26,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166592) roll requested 2024-11-13T14:29:26,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166600 2024-11-13T14:29:26,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166600 2024-11-13T14:29:26,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166600) roll requested 2024-11-13T14:29:26,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166607 2024-11-13T14:29:26,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166607 2024-11-13T14:29:26,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166607) roll requested 2024-11-13T14:29:26,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166613 2024-11-13T14:29:26,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166613 2024-11-13T14:29:26,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166613) roll requested 2024-11-13T14:29:26,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166623 2024-11-13T14:29:26,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166623 2024-11-13T14:29:26,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166623) roll requested 2024-11-13T14:29:26,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166629 2024-11-13T14:29:26,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166629 2024-11-13T14:29:26,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166629) roll requested 2024-11-13T14:29:26,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166634 2024-11-13T14:29:26,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166634 2024-11-13T14:29:26,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166634) roll requested 2024-11-13T14:29:26,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166639 2024-11-13T14:29:26,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166639 2024-11-13T14:29:26,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166639) roll requested 2024-11-13T14:29:26,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166645 2024-11-13T14:29:26,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166645 2024-11-13T14:29:26,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166645) roll requested 2024-11-13T14:29:26,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166652 2024-11-13T14:29:26,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166652 2024-11-13T14:29:26,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166652) roll requested 2024-11-13T14:29:26,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166659 2024-11-13T14:29:26,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166659 2024-11-13T14:29:26,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166659) roll requested 2024-11-13T14:29:26,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166667 2024-11-13T14:29:26,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166667 2024-11-13T14:29:26,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166667) roll requested 2024-11-13T14:29:26,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166674 2024-11-13T14:29:26,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166674 2024-11-13T14:29:26,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166674) roll requested 2024-11-13T14:29:26,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166683 2024-11-13T14:29:26,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166683 2024-11-13T14:29:26,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166683) roll requested 2024-11-13T14:29:26,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166693 2024-11-13T14:29:26,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166693 2024-11-13T14:29:26,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166693) roll requested 2024-11-13T14:29:26,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166703 2024-11-13T14:29:26,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166703 2024-11-13T14:29:26,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166703) roll requested 2024-11-13T14:29:26,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166713 2024-11-13T14:29:26,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166713 2024-11-13T14:29:26,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166713) roll requested 2024-11-13T14:29:26,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166721 2024-11-13T14:29:26,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166721 2024-11-13T14:29:26,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166721) roll requested 2024-11-13T14:29:26,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166727 2024-11-13T14:29:26,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166727 2024-11-13T14:29:26,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166727) roll requested 2024-11-13T14:29:26,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166732 2024-11-13T14:29:26,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166732 2024-11-13T14:29:26,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166732) roll requested 2024-11-13T14:29:26,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166736 2024-11-13T14:29:26,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166736 2024-11-13T14:29:26,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166736) roll requested 2024-11-13T14:29:26,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166742 2024-11-13T14:29:26,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166742 2024-11-13T14:29:26,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166742) roll requested 2024-11-13T14:29:26,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166750 2024-11-13T14:29:26,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166750 2024-11-13T14:29:26,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166750) roll requested 2024-11-13T14:29:26,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166760 2024-11-13T14:29:26,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166760 2024-11-13T14:29:26,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166760) roll requested 2024-11-13T14:29:26,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166769 2024-11-13T14:29:26,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166769 2024-11-13T14:29:26,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166769) roll requested 2024-11-13T14:29:26,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166774 2024-11-13T14:29:26,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166774 2024-11-13T14:29:26,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166774) roll requested 2024-11-13T14:29:26,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166785 2024-11-13T14:29:26,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166785 2024-11-13T14:29:26,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166785) roll requested 2024-11-13T14:29:26,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166791 2024-11-13T14:29:26,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166791 2024-11-13T14:29:26,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166791) roll requested 2024-11-13T14:29:26,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166797 2024-11-13T14:29:26,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166797 2024-11-13T14:29:26,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166797) roll requested 2024-11-13T14:29:26,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166802 2024-11-13T14:29:26,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166802 2024-11-13T14:29:26,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166802) roll requested 2024-11-13T14:29:26,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166808 2024-11-13T14:29:26,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166808 2024-11-13T14:29:26,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166808) roll requested 2024-11-13T14:29:26,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166816 2024-11-13T14:29:26,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166816 2024-11-13T14:29:26,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166816) roll requested 2024-11-13T14:29:26,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166822 2024-11-13T14:29:26,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166822 2024-11-13T14:29:26,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166822) roll requested 2024-11-13T14:29:26,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166828 2024-11-13T14:29:26,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166828 2024-11-13T14:29:26,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166828) roll requested 2024-11-13T14:29:26,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166834 2024-11-13T14:29:26,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166834 2024-11-13T14:29:26,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166834) roll requested 2024-11-13T14:29:26,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166840 2024-11-13T14:29:26,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166840 2024-11-13T14:29:26,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166840) roll requested 2024-11-13T14:29:26,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166846 2024-11-13T14:29:26,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166846 2024-11-13T14:29:26,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166846) roll requested 2024-11-13T14:29:26,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166851 2024-11-13T14:29:26,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166851 2024-11-13T14:29:26,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166851) roll requested 2024-11-13T14:29:26,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166856 2024-11-13T14:29:26,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166856 2024-11-13T14:29:26,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166856) roll requested 2024-11-13T14:29:26,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166862 2024-11-13T14:29:26,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166862 2024-11-13T14:29:26,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166862) roll requested 2024-11-13T14:29:26,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166868 2024-11-13T14:29:26,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166868 2024-11-13T14:29:26,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166868) roll requested 2024-11-13T14:29:26,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166874 2024-11-13T14:29:26,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166874 2024-11-13T14:29:26,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166874) roll requested 2024-11-13T14:29:26,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166880 2024-11-13T14:29:26,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166880 2024-11-13T14:29:26,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166880) roll requested 2024-11-13T14:29:26,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166886 2024-11-13T14:29:26,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166886 2024-11-13T14:29:26,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166886) roll requested 2024-11-13T14:29:26,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166891 2024-11-13T14:29:26,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166891 2024-11-13T14:29:26,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166891) roll requested 2024-11-13T14:29:26,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166896 2024-11-13T14:29:26,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166896 2024-11-13T14:29:26,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166896) roll requested 2024-11-13T14:29:26,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166902 2024-11-13T14:29:26,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166902 2024-11-13T14:29:26,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166902) roll requested 2024-11-13T14:29:26,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166908 2024-11-13T14:29:26,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166908 2024-11-13T14:29:26,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166908) roll requested 2024-11-13T14:29:26,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166914 2024-11-13T14:29:26,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166914 2024-11-13T14:29:26,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166914) roll requested 2024-11-13T14:29:26,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166922 2024-11-13T14:29:26,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166922 2024-11-13T14:29:26,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166922) roll requested 2024-11-13T14:29:26,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166932 2024-11-13T14:29:26,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166932 2024-11-13T14:29:26,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166932) roll requested 2024-11-13T14:29:26,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166938 2024-11-13T14:29:26,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166938 2024-11-13T14:29:26,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166938) roll requested 2024-11-13T14:29:26,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166945 2024-11-13T14:29:26,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166945 2024-11-13T14:29:26,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166945) roll requested 2024-11-13T14:29:26,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166956 2024-11-13T14:29:26,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166956 2024-11-13T14:29:26,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166956) roll requested 2024-11-13T14:29:26,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166962 2024-11-13T14:29:26,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166962 2024-11-13T14:29:26,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166962) roll requested 2024-11-13T14:29:26,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166968 2024-11-13T14:29:26,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166968 2024-11-13T14:29:26,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166968) roll requested 2024-11-13T14:29:26,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166973 2024-11-13T14:29:26,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166973 2024-11-13T14:29:26,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166973) roll requested 2024-11-13T14:29:26,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166978 2024-11-13T14:29:26,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166978 2024-11-13T14:29:26,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166978) roll requested 2024-11-13T14:29:26,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166984 2024-11-13T14:29:26,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166984 2024-11-13T14:29:26,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166984) roll requested 2024-11-13T14:29:26,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166990 2024-11-13T14:29:26,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166990 2024-11-13T14:29:26,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:26,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:26,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166990) roll requested 2024-11-13T14:29:26,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508166995 2024-11-13T14:29:26,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:26,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166995 2024-11-13T14:29:27,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508166995) roll requested 2024-11-13T14:29:27,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167000 2024-11-13T14:29:27,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508166995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167000 2024-11-13T14:29:27,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167000) roll requested 2024-11-13T14:29:27,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167007 2024-11-13T14:29:27,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167007 2024-11-13T14:29:27,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167007) roll requested 2024-11-13T14:29:27,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167014 2024-11-13T14:29:27,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167014 2024-11-13T14:29:27,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167014) roll requested 2024-11-13T14:29:27,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167020 2024-11-13T14:29:27,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167020 2024-11-13T14:29:27,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167020) roll requested 2024-11-13T14:29:27,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167025 2024-11-13T14:29:27,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167025 2024-11-13T14:29:27,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167025) roll requested 2024-11-13T14:29:27,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167033 2024-11-13T14:29:27,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167033 2024-11-13T14:29:27,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167033) roll requested 2024-11-13T14:29:27,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167040 2024-11-13T14:29:27,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167040 2024-11-13T14:29:27,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167040) roll requested 2024-11-13T14:29:27,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167049 2024-11-13T14:29:27,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167049 2024-11-13T14:29:27,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167049) roll requested 2024-11-13T14:29:27,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167057 2024-11-13T14:29:27,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167057 2024-11-13T14:29:27,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167057) roll requested 2024-11-13T14:29:27,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167064 2024-11-13T14:29:27,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167064 2024-11-13T14:29:27,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167064) roll requested 2024-11-13T14:29:27,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167071 2024-11-13T14:29:27,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167071 2024-11-13T14:29:27,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167071) roll requested 2024-11-13T14:29:27,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167077 2024-11-13T14:29:27,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167077 2024-11-13T14:29:27,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167077) roll requested 2024-11-13T14:29:27,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167084 2024-11-13T14:29:27,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167084 2024-11-13T14:29:27,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167084) roll requested 2024-11-13T14:29:27,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167090 2024-11-13T14:29:27,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167090 2024-11-13T14:29:27,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167090) roll requested 2024-11-13T14:29:27,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167097 2024-11-13T14:29:27,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167097 2024-11-13T14:29:27,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167097) roll requested 2024-11-13T14:29:27,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167105 2024-11-13T14:29:27,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167105 2024-11-13T14:29:27,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167105) roll requested 2024-11-13T14:29:27,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167114 2024-11-13T14:29:27,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167114 2024-11-13T14:29:27,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167114) roll requested 2024-11-13T14:29:27,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167122 2024-11-13T14:29:27,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167122 2024-11-13T14:29:27,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167122) roll requested 2024-11-13T14:29:27,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167131 2024-11-13T14:29:27,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167131 2024-11-13T14:29:27,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167131) roll requested 2024-11-13T14:29:27,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167142 2024-11-13T14:29:27,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167142 2024-11-13T14:29:27,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167142) roll requested 2024-11-13T14:29:27,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167151 2024-11-13T14:29:27,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167151 2024-11-13T14:29:27,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167151) roll requested 2024-11-13T14:29:27,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167158 2024-11-13T14:29:27,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167158 2024-11-13T14:29:27,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167158) roll requested 2024-11-13T14:29:27,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167165 2024-11-13T14:29:27,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167165 2024-11-13T14:29:27,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167165) roll requested 2024-11-13T14:29:27,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167174 2024-11-13T14:29:27,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167174 2024-11-13T14:29:27,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167174) roll requested 2024-11-13T14:29:27,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167182 2024-11-13T14:29:27,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167182 2024-11-13T14:29:27,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167182) roll requested 2024-11-13T14:29:27,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167193 2024-11-13T14:29:27,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167193 2024-11-13T14:29:27,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167193) roll requested 2024-11-13T14:29:27,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167205 2024-11-13T14:29:27,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167205 2024-11-13T14:29:27,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167205) roll requested 2024-11-13T14:29:27,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167211 2024-11-13T14:29:27,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167211 2024-11-13T14:29:27,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167211) roll requested 2024-11-13T14:29:27,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167216 2024-11-13T14:29:27,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167216 2024-11-13T14:29:27,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167216) roll requested 2024-11-13T14:29:27,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167222 2024-11-13T14:29:27,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167222 2024-11-13T14:29:27,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167222) roll requested 2024-11-13T14:29:27,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167229 2024-11-13T14:29:27,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167229 2024-11-13T14:29:27,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167229) roll requested 2024-11-13T14:29:27,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167237 2024-11-13T14:29:27,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167237 2024-11-13T14:29:27,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167237) roll requested 2024-11-13T14:29:27,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167246 2024-11-13T14:29:27,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167246 2024-11-13T14:29:27,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167246) roll requested 2024-11-13T14:29:27,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167255 2024-11-13T14:29:27,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167255 2024-11-13T14:29:27,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167255) roll requested 2024-11-13T14:29:27,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167264 2024-11-13T14:29:27,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167264 2024-11-13T14:29:27,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167264) roll requested 2024-11-13T14:29:27,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167275 2024-11-13T14:29:27,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167275 2024-11-13T14:29:27,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167275) roll requested 2024-11-13T14:29:27,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167285 2024-11-13T14:29:27,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167285 2024-11-13T14:29:27,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167285) roll requested 2024-11-13T14:29:27,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167293 2024-11-13T14:29:27,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167293 2024-11-13T14:29:27,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167293) roll requested 2024-11-13T14:29:27,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167301 2024-11-13T14:29:27,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167301 2024-11-13T14:29:27,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167301) roll requested 2024-11-13T14:29:27,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167308 2024-11-13T14:29:27,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167308 2024-11-13T14:29:27,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167308) roll requested 2024-11-13T14:29:27,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167318 2024-11-13T14:29:27,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167318 2024-11-13T14:29:27,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167318) roll requested 2024-11-13T14:29:27,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167327 2024-11-13T14:29:27,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167327 2024-11-13T14:29:27,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167327) roll requested 2024-11-13T14:29:27,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167335 2024-11-13T14:29:27,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167335 2024-11-13T14:29:27,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167335) roll requested 2024-11-13T14:29:27,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167343 2024-11-13T14:29:27,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167343 2024-11-13T14:29:27,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167343) roll requested 2024-11-13T14:29:27,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167350 2024-11-13T14:29:27,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167350 2024-11-13T14:29:27,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167350) roll requested 2024-11-13T14:29:27,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167356 2024-11-13T14:29:27,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167356 2024-11-13T14:29:27,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167356) roll requested 2024-11-13T14:29:27,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167364 2024-11-13T14:29:27,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167364 2024-11-13T14:29:27,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167364) roll requested 2024-11-13T14:29:27,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167372 2024-11-13T14:29:27,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167372 2024-11-13T14:29:27,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167372) roll requested 2024-11-13T14:29:27,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167380 2024-11-13T14:29:27,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167380 2024-11-13T14:29:27,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167380) roll requested 2024-11-13T14:29:27,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167389 2024-11-13T14:29:27,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167389 2024-11-13T14:29:27,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167389) roll requested 2024-11-13T14:29:27,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167396 2024-11-13T14:29:27,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167396 2024-11-13T14:29:27,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167396) roll requested 2024-11-13T14:29:27,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167405 2024-11-13T14:29:27,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167405 2024-11-13T14:29:27,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167405) roll requested 2024-11-13T14:29:27,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167414 2024-11-13T14:29:27,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167414 2024-11-13T14:29:27,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167414) roll requested 2024-11-13T14:29:27,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167422 2024-11-13T14:29:27,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167422 2024-11-13T14:29:27,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167422) roll requested 2024-11-13T14:29:27,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167430 2024-11-13T14:29:27,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167430 2024-11-13T14:29:27,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167430) roll requested 2024-11-13T14:29:27,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167440 2024-11-13T14:29:27,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167440 2024-11-13T14:29:27,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167440) roll requested 2024-11-13T14:29:27,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167452 2024-11-13T14:29:27,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167452 2024-11-13T14:29:27,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167452) roll requested 2024-11-13T14:29:27,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167460 2024-11-13T14:29:27,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167460 2024-11-13T14:29:27,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167460) roll requested 2024-11-13T14:29:27,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167465 2024-11-13T14:29:27,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167465 2024-11-13T14:29:27,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167465) roll requested 2024-11-13T14:29:27,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167470 2024-11-13T14:29:27,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167470 2024-11-13T14:29:27,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167470) roll requested 2024-11-13T14:29:27,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167476 2024-11-13T14:29:27,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167476 2024-11-13T14:29:27,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167476) roll requested 2024-11-13T14:29:27,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167483 2024-11-13T14:29:27,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167483 2024-11-13T14:29:27,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167483) roll requested 2024-11-13T14:29:27,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167489 2024-11-13T14:29:27,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167489 2024-11-13T14:29:27,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167489) roll requested 2024-11-13T14:29:27,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167496 2024-11-13T14:29:27,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167496 2024-11-13T14:29:27,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167496) roll requested 2024-11-13T14:29:27,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167503 2024-11-13T14:29:27,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167503 2024-11-13T14:29:27,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167503) roll requested 2024-11-13T14:29:27,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167509 2024-11-13T14:29:27,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167509 2024-11-13T14:29:27,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167509) roll requested 2024-11-13T14:29:27,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167519 2024-11-13T14:29:27,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167519 2024-11-13T14:29:27,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167519) roll requested 2024-11-13T14:29:27,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167524 2024-11-13T14:29:27,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167524 2024-11-13T14:29:27,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167524) roll requested 2024-11-13T14:29:27,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167530 2024-11-13T14:29:27,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167530 2024-11-13T14:29:27,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167530) roll requested 2024-11-13T14:29:27,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167536 2024-11-13T14:29:27,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167536 2024-11-13T14:29:27,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167536) roll requested 2024-11-13T14:29:27,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167541 2024-11-13T14:29:27,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167541 2024-11-13T14:29:27,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167541) roll requested 2024-11-13T14:29:27,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167547 2024-11-13T14:29:27,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167547 2024-11-13T14:29:27,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167547) roll requested 2024-11-13T14:29:27,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167553 2024-11-13T14:29:27,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167553 2024-11-13T14:29:27,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167553) roll requested 2024-11-13T14:29:27,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167560 2024-11-13T14:29:27,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167560 2024-11-13T14:29:27,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167560) roll requested 2024-11-13T14:29:27,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167568 2024-11-13T14:29:27,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167568 2024-11-13T14:29:27,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167568) roll requested 2024-11-13T14:29:27,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167577 2024-11-13T14:29:27,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167577 2024-11-13T14:29:27,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167577) roll requested 2024-11-13T14:29:27,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167584 2024-11-13T14:29:27,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167584 2024-11-13T14:29:27,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167584) roll requested 2024-11-13T14:29:27,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167592 2024-11-13T14:29:27,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167592 2024-11-13T14:29:27,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167592) roll requested 2024-11-13T14:29:27,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167601 2024-11-13T14:29:27,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167601 2024-11-13T14:29:27,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167601) roll requested 2024-11-13T14:29:27,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167607 2024-11-13T14:29:27,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167607 2024-11-13T14:29:27,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167607) roll requested 2024-11-13T14:29:27,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167612 2024-11-13T14:29:27,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167612 2024-11-13T14:29:27,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167612) roll requested 2024-11-13T14:29:27,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167619 2024-11-13T14:29:27,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167619 2024-11-13T14:29:27,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167619) roll requested 2024-11-13T14:29:27,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167627 2024-11-13T14:29:27,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167627 2024-11-13T14:29:27,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167627) roll requested 2024-11-13T14:29:27,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167635 2024-11-13T14:29:27,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167635 2024-11-13T14:29:27,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167635) roll requested 2024-11-13T14:29:27,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167645 2024-11-13T14:29:27,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167645 2024-11-13T14:29:27,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167645) roll requested 2024-11-13T14:29:27,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167650 2024-11-13T14:29:27,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167650 2024-11-13T14:29:27,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167650) roll requested 2024-11-13T14:29:27,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167656 2024-11-13T14:29:27,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167656 2024-11-13T14:29:27,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167656) roll requested 2024-11-13T14:29:27,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167663 2024-11-13T14:29:27,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167663 2024-11-13T14:29:27,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167663) roll requested 2024-11-13T14:29:27,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167671 2024-11-13T14:29:27,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167671 2024-11-13T14:29:27,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167671) roll requested 2024-11-13T14:29:27,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167680 2024-11-13T14:29:27,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167680 2024-11-13T14:29:27,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167680) roll requested 2024-11-13T14:29:27,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167689 2024-11-13T14:29:27,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167689 2024-11-13T14:29:27,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167689) roll requested 2024-11-13T14:29:27,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167695 2024-11-13T14:29:27,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167695 2024-11-13T14:29:27,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167695) roll requested 2024-11-13T14:29:27,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167702 2024-11-13T14:29:27,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167702 2024-11-13T14:29:27,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167702) roll requested 2024-11-13T14:29:27,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167707 2024-11-13T14:29:27,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167707 2024-11-13T14:29:27,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167707) roll requested 2024-11-13T14:29:27,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167711 2024-11-13T14:29:27,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167711 2024-11-13T14:29:27,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167711) roll requested 2024-11-13T14:29:27,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167718 2024-11-13T14:29:27,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167718 2024-11-13T14:29:27,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167718) roll requested 2024-11-13T14:29:27,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167725 2024-11-13T14:29:27,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167725 2024-11-13T14:29:27,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167725) roll requested 2024-11-13T14:29:27,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167733 2024-11-13T14:29:27,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167733 2024-11-13T14:29:27,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167733) roll requested 2024-11-13T14:29:27,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167742 2024-11-13T14:29:27,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167742 2024-11-13T14:29:27,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167742) roll requested 2024-11-13T14:29:27,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167751 2024-11-13T14:29:27,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167751 2024-11-13T14:29:27,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167751) roll requested 2024-11-13T14:29:27,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167761 2024-11-13T14:29:27,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167761 2024-11-13T14:29:27,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167761) roll requested 2024-11-13T14:29:27,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167767 2024-11-13T14:29:27,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167767 2024-11-13T14:29:27,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167767) roll requested 2024-11-13T14:29:27,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167772 2024-11-13T14:29:27,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167772 2024-11-13T14:29:27,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167772) roll requested 2024-11-13T14:29:27,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167780 2024-11-13T14:29:27,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167780 2024-11-13T14:29:27,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167780) roll requested 2024-11-13T14:29:27,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167786 2024-11-13T14:29:27,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167786 2024-11-13T14:29:27,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167786) roll requested 2024-11-13T14:29:27,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167795 2024-11-13T14:29:27,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167795 2024-11-13T14:29:27,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167795) roll requested 2024-11-13T14:29:27,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167800 2024-11-13T14:29:27,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167800 2024-11-13T14:29:27,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167800) roll requested 2024-11-13T14:29:27,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167805 2024-11-13T14:29:27,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167805 2024-11-13T14:29:27,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167805) roll requested 2024-11-13T14:29:27,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167811 2024-11-13T14:29:27,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167811 2024-11-13T14:29:27,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167811) roll requested 2024-11-13T14:29:27,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167817 2024-11-13T14:29:27,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167817 2024-11-13T14:29:27,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167817) roll requested 2024-11-13T14:29:27,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167826 2024-11-13T14:29:27,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167826 2024-11-13T14:29:27,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167826) roll requested 2024-11-13T14:29:27,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167835 2024-11-13T14:29:27,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167835 2024-11-13T14:29:27,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167835) roll requested 2024-11-13T14:29:27,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167845 2024-11-13T14:29:27,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167845 2024-11-13T14:29:27,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167845) roll requested 2024-11-13T14:29:27,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167855 2024-11-13T14:29:27,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167855 2024-11-13T14:29:27,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167855) roll requested 2024-11-13T14:29:27,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167864 2024-11-13T14:29:27,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167864 2024-11-13T14:29:27,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167864) roll requested 2024-11-13T14:29:27,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167870 2024-11-13T14:29:27,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167870 2024-11-13T14:29:27,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167870) roll requested 2024-11-13T14:29:27,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167875 2024-11-13T14:29:27,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167875 2024-11-13T14:29:27,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167875) roll requested 2024-11-13T14:29:27,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167882 2024-11-13T14:29:27,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167882 2024-11-13T14:29:27,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167882) roll requested 2024-11-13T14:29:27,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167889 2024-11-13T14:29:27,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167889 2024-11-13T14:29:27,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167889) roll requested 2024-11-13T14:29:27,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167897 2024-11-13T14:29:27,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167897 2024-11-13T14:29:27,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167897) roll requested 2024-11-13T14:29:27,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167905 2024-11-13T14:29:27,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167905 2024-11-13T14:29:27,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167905) roll requested 2024-11-13T14:29:27,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167912 2024-11-13T14:29:27,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167912 2024-11-13T14:29:27,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167912) roll requested 2024-11-13T14:29:27,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167920 2024-11-13T14:29:27,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167920 2024-11-13T14:29:27,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167920) roll requested 2024-11-13T14:29:27,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167928 2024-11-13T14:29:27,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167928 2024-11-13T14:29:27,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167928) roll requested 2024-11-13T14:29:27,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167934 2024-11-13T14:29:27,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167934 2024-11-13T14:29:27,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167934) roll requested 2024-11-13T14:29:27,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167939 2024-11-13T14:29:27,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167939 2024-11-13T14:29:27,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167939) roll requested 2024-11-13T14:29:27,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167947 2024-11-13T14:29:27,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167947 2024-11-13T14:29:27,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167947) roll requested 2024-11-13T14:29:27,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167955 2024-11-13T14:29:27,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167955 2024-11-13T14:29:27,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167955) roll requested 2024-11-13T14:29:27,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167961 2024-11-13T14:29:27,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167961 2024-11-13T14:29:27,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167961) roll requested 2024-11-13T14:29:27,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167967 2024-11-13T14:29:27,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167967 2024-11-13T14:29:27,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167967) roll requested 2024-11-13T14:29:27,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167974 2024-11-13T14:29:27,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167974 2024-11-13T14:29:27,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167974) roll requested 2024-11-13T14:29:27,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167982 2024-11-13T14:29:27,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167982 2024-11-13T14:29:27,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167982) roll requested 2024-11-13T14:29:27,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167991 2024-11-13T14:29:27,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:27,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167991 2024-11-13T14:29:27,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:27,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:27,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167991) roll requested 2024-11-13T14:29:27,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508167999 2024-11-13T14:29:28,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167999 2024-11-13T14:29:28,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508167999) roll requested 2024-11-13T14:29:28,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168005 2024-11-13T14:29:28,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508167999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168005 2024-11-13T14:29:28,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168005) roll requested 2024-11-13T14:29:28,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168012 2024-11-13T14:29:28,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168012 2024-11-13T14:29:28,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168012) roll requested 2024-11-13T14:29:28,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168020 2024-11-13T14:29:28,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168020 2024-11-13T14:29:28,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168020) roll requested 2024-11-13T14:29:28,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168029 2024-11-13T14:29:28,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168029 2024-11-13T14:29:28,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168029) roll requested 2024-11-13T14:29:28,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168036 2024-11-13T14:29:28,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168036 2024-11-13T14:29:28,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168036) roll requested 2024-11-13T14:29:28,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168044 2024-11-13T14:29:28,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168044 2024-11-13T14:29:28,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168044) roll requested 2024-11-13T14:29:28,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168052 2024-11-13T14:29:28,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168052 2024-11-13T14:29:28,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168052) roll requested 2024-11-13T14:29:28,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168061 2024-11-13T14:29:28,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168061 2024-11-13T14:29:28,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168061) roll requested 2024-11-13T14:29:28,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168068 2024-11-13T14:29:28,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168068 2024-11-13T14:29:28,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168068) roll requested 2024-11-13T14:29:28,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168076 2024-11-13T14:29:28,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168076 2024-11-13T14:29:28,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168076) roll requested 2024-11-13T14:29:28,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168083 2024-11-13T14:29:28,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168083 2024-11-13T14:29:28,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168083) roll requested 2024-11-13T14:29:28,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168091 2024-11-13T14:29:28,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168091 2024-11-13T14:29:28,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168091) roll requested 2024-11-13T14:29:28,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168098 2024-11-13T14:29:28,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168098 2024-11-13T14:29:28,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168098) roll requested 2024-11-13T14:29:28,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168107 2024-11-13T14:29:28,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168107 2024-11-13T14:29:28,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168107) roll requested 2024-11-13T14:29:28,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168116 2024-11-13T14:29:28,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168116 2024-11-13T14:29:28,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168116) roll requested 2024-11-13T14:29:28,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168125 2024-11-13T14:29:28,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168125 2024-11-13T14:29:28,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168125) roll requested 2024-11-13T14:29:28,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168134 2024-11-13T14:29:28,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168134 2024-11-13T14:29:28,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168134) roll requested 2024-11-13T14:29:28,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168151 2024-11-13T14:29:28,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168151 2024-11-13T14:29:28,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168151) roll requested 2024-11-13T14:29:28,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168157 2024-11-13T14:29:28,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168157 2024-11-13T14:29:28,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168157) roll requested 2024-11-13T14:29:28,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168162 2024-11-13T14:29:28,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168162 2024-11-13T14:29:28,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168162) roll requested 2024-11-13T14:29:28,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168169 2024-11-13T14:29:28,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168169 2024-11-13T14:29:28,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168169) roll requested 2024-11-13T14:29:28,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168175 2024-11-13T14:29:28,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168175 2024-11-13T14:29:28,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168175) roll requested 2024-11-13T14:29:28,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168182 2024-11-13T14:29:28,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168182 2024-11-13T14:29:28,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168182) roll requested 2024-11-13T14:29:28,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168190 2024-11-13T14:29:28,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168190 2024-11-13T14:29:28,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168190) roll requested 2024-11-13T14:29:28,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168197 2024-11-13T14:29:28,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168197 2024-11-13T14:29:28,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168197) roll requested 2024-11-13T14:29:28,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168204 2024-11-13T14:29:28,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168204 2024-11-13T14:29:28,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168204) roll requested 2024-11-13T14:29:28,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168211 2024-11-13T14:29:28,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168211 2024-11-13T14:29:28,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168211) roll requested 2024-11-13T14:29:28,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168219 2024-11-13T14:29:28,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168219 2024-11-13T14:29:28,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168219) roll requested 2024-11-13T14:29:28,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168226 2024-11-13T14:29:28,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168226 2024-11-13T14:29:28,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168226) roll requested 2024-11-13T14:29:28,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168233 2024-11-13T14:29:28,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168233 2024-11-13T14:29:28,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168233) roll requested 2024-11-13T14:29:28,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168242 2024-11-13T14:29:28,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168242 2024-11-13T14:29:28,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168242) roll requested 2024-11-13T14:29:28,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168250 2024-11-13T14:29:28,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168250 2024-11-13T14:29:28,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168250) roll requested 2024-11-13T14:29:28,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168257 2024-11-13T14:29:28,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168257 2024-11-13T14:29:28,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168257) roll requested 2024-11-13T14:29:28,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168266 2024-11-13T14:29:28,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168266 2024-11-13T14:29:28,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168266) roll requested 2024-11-13T14:29:28,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168276 2024-11-13T14:29:28,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168276 2024-11-13T14:29:28,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168276) roll requested 2024-11-13T14:29:28,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168286 2024-11-13T14:29:28,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168286 2024-11-13T14:29:28,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168286) roll requested 2024-11-13T14:29:28,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168297 2024-11-13T14:29:28,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168297 2024-11-13T14:29:28,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168297) roll requested 2024-11-13T14:29:28,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168305 2024-11-13T14:29:28,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168305 2024-11-13T14:29:28,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168305) roll requested 2024-11-13T14:29:28,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168313 2024-11-13T14:29:28,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168313 2024-11-13T14:29:28,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168313) roll requested 2024-11-13T14:29:28,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168323 2024-11-13T14:29:28,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168323 2024-11-13T14:29:28,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168323) roll requested 2024-11-13T14:29:28,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168329 2024-11-13T14:29:28,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168329 2024-11-13T14:29:28,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168329) roll requested 2024-11-13T14:29:28,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168336 2024-11-13T14:29:28,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168336 2024-11-13T14:29:28,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168336) roll requested 2024-11-13T14:29:28,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168345 2024-11-13T14:29:28,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168345 2024-11-13T14:29:28,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168345) roll requested 2024-11-13T14:29:28,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168352 2024-11-13T14:29:28,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168352 2024-11-13T14:29:28,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168352) roll requested 2024-11-13T14:29:28,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168361 2024-11-13T14:29:28,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168361 2024-11-13T14:29:28,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168361) roll requested 2024-11-13T14:29:28,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168367 2024-11-13T14:29:28,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168367 2024-11-13T14:29:28,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168367) roll requested 2024-11-13T14:29:28,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168374 2024-11-13T14:29:28,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168374 2024-11-13T14:29:28,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168374) roll requested 2024-11-13T14:29:28,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168383 2024-11-13T14:29:28,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168383 2024-11-13T14:29:28,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168383) roll requested 2024-11-13T14:29:28,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168392 2024-11-13T14:29:28,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168392 2024-11-13T14:29:28,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168392) roll requested 2024-11-13T14:29:28,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168400 2024-11-13T14:29:28,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168400 2024-11-13T14:29:28,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168400) roll requested 2024-11-13T14:29:28,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168408 2024-11-13T14:29:28,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168408 2024-11-13T14:29:28,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168408) roll requested 2024-11-13T14:29:28,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168415 2024-11-13T14:29:28,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168415 2024-11-13T14:29:28,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168415) roll requested 2024-11-13T14:29:28,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168422 2024-11-13T14:29:28,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168422 2024-11-13T14:29:28,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168422) roll requested 2024-11-13T14:29:28,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168428 2024-11-13T14:29:28,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168428 2024-11-13T14:29:28,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168428) roll requested 2024-11-13T14:29:28,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168435 2024-11-13T14:29:28,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168435 2024-11-13T14:29:28,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168435) roll requested 2024-11-13T14:29:28,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168440 2024-11-13T14:29:28,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168440 2024-11-13T14:29:28,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168440) roll requested 2024-11-13T14:29:28,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168446 2024-11-13T14:29:28,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168446 2024-11-13T14:29:28,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168446) roll requested 2024-11-13T14:29:28,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168455 2024-11-13T14:29:28,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168455 2024-11-13T14:29:28,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168455) roll requested 2024-11-13T14:29:28,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168461 2024-11-13T14:29:28,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168461 2024-11-13T14:29:28,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168461) roll requested 2024-11-13T14:29:28,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168467 2024-11-13T14:29:28,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168467 2024-11-13T14:29:28,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168467) roll requested 2024-11-13T14:29:28,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168473 2024-11-13T14:29:28,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168473 2024-11-13T14:29:28,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168473) roll requested 2024-11-13T14:29:28,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168478 2024-11-13T14:29:28,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168478 2024-11-13T14:29:28,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168478) roll requested 2024-11-13T14:29:28,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168485 2024-11-13T14:29:28,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168485 2024-11-13T14:29:28,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168485) roll requested 2024-11-13T14:29:28,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168492 2024-11-13T14:29:28,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168492 2024-11-13T14:29:28,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168492) roll requested 2024-11-13T14:29:28,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168497 2024-11-13T14:29:28,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168497 2024-11-13T14:29:28,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168497) roll requested 2024-11-13T14:29:28,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168503 2024-11-13T14:29:28,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168503 2024-11-13T14:29:28,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168503) roll requested 2024-11-13T14:29:28,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168508 2024-11-13T14:29:28,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168508 2024-11-13T14:29:28,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168508) roll requested 2024-11-13T14:29:28,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168514 2024-11-13T14:29:28,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168514 2024-11-13T14:29:28,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168514) roll requested 2024-11-13T14:29:28,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168520 2024-11-13T14:29:28,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168520 2024-11-13T14:29:28,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168520) roll requested 2024-11-13T14:29:28,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168528 2024-11-13T14:29:28,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168528 2024-11-13T14:29:28,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168528) roll requested 2024-11-13T14:29:28,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168533 2024-11-13T14:29:28,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168533 2024-11-13T14:29:28,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168533) roll requested 2024-11-13T14:29:28,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168539 2024-11-13T14:29:28,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168539 2024-11-13T14:29:28,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168539) roll requested 2024-11-13T14:29:28,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168544 2024-11-13T14:29:28,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168544 2024-11-13T14:29:28,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168544) roll requested 2024-11-13T14:29:28,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168549 2024-11-13T14:29:28,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168549 2024-11-13T14:29:28,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168549) roll requested 2024-11-13T14:29:28,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168554 2024-11-13T14:29:28,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168554 2024-11-13T14:29:28,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168554) roll requested 2024-11-13T14:29:28,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168559 2024-11-13T14:29:28,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168559 2024-11-13T14:29:28,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168559) roll requested 2024-11-13T14:29:28,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168564 2024-11-13T14:29:28,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168564 2024-11-13T14:29:28,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168564) roll requested 2024-11-13T14:29:28,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168570 2024-11-13T14:29:28,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168570 2024-11-13T14:29:28,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168570) roll requested 2024-11-13T14:29:28,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168576 2024-11-13T14:29:28,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168576 2024-11-13T14:29:28,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168576) roll requested 2024-11-13T14:29:28,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168582 2024-11-13T14:29:28,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168582 2024-11-13T14:29:28,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168582) roll requested 2024-11-13T14:29:28,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168587 2024-11-13T14:29:28,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168587 2024-11-13T14:29:28,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168587) roll requested 2024-11-13T14:29:28,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168592 2024-11-13T14:29:28,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168592 2024-11-13T14:29:28,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168592) roll requested 2024-11-13T14:29:28,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168598 2024-11-13T14:29:28,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168598 2024-11-13T14:29:28,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168598) roll requested 2024-11-13T14:29:28,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168603 2024-11-13T14:29:28,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168603 2024-11-13T14:29:28,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168603) roll requested 2024-11-13T14:29:28,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168608 2024-11-13T14:29:28,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168608 2024-11-13T14:29:28,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168608) roll requested 2024-11-13T14:29:28,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168614 2024-11-13T14:29:28,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168614 2024-11-13T14:29:28,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168614) roll requested 2024-11-13T14:29:28,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168619 2024-11-13T14:29:28,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168619 2024-11-13T14:29:28,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168619) roll requested 2024-11-13T14:29:28,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168624 2024-11-13T14:29:28,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168624 2024-11-13T14:29:28,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168624) roll requested 2024-11-13T14:29:28,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168629 2024-11-13T14:29:28,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168629 2024-11-13T14:29:28,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168629) roll requested 2024-11-13T14:29:28,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168635 2024-11-13T14:29:28,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168635 2024-11-13T14:29:28,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168635) roll requested 2024-11-13T14:29:28,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168640 2024-11-13T14:29:28,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168640 2024-11-13T14:29:28,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168640) roll requested 2024-11-13T14:29:28,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168646 2024-11-13T14:29:28,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168646 2024-11-13T14:29:28,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168646) roll requested 2024-11-13T14:29:28,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168651 2024-11-13T14:29:28,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168651 2024-11-13T14:29:28,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168651) roll requested 2024-11-13T14:29:28,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168656 2024-11-13T14:29:28,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168656 2024-11-13T14:29:28,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168656) roll requested 2024-11-13T14:29:28,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168661 2024-11-13T14:29:28,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168661 2024-11-13T14:29:28,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168661) roll requested 2024-11-13T14:29:28,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168668 2024-11-13T14:29:28,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168668 2024-11-13T14:29:28,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168668) roll requested 2024-11-13T14:29:28,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168675 2024-11-13T14:29:28,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168675 2024-11-13T14:29:28,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168675) roll requested 2024-11-13T14:29:28,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168683 2024-11-13T14:29:28,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168683 2024-11-13T14:29:28,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168683) roll requested 2024-11-13T14:29:28,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168694 2024-11-13T14:29:28,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168694 2024-11-13T14:29:28,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168694) roll requested 2024-11-13T14:29:28,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168704 2024-11-13T14:29:28,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168704 2024-11-13T14:29:28,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168704) roll requested 2024-11-13T14:29:28,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168713 2024-11-13T14:29:28,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168713 2024-11-13T14:29:28,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168713) roll requested 2024-11-13T14:29:28,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168721 2024-11-13T14:29:28,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168721 2024-11-13T14:29:28,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168721) roll requested 2024-11-13T14:29:28,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168729 2024-11-13T14:29:28,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168729 2024-11-13T14:29:28,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168729) roll requested 2024-11-13T14:29:28,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168738 2024-11-13T14:29:28,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168738 2024-11-13T14:29:28,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168738) roll requested 2024-11-13T14:29:28,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168748 2024-11-13T14:29:28,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168748 2024-11-13T14:29:28,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168748) roll requested 2024-11-13T14:29:28,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168756 2024-11-13T14:29:28,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168756 2024-11-13T14:29:28,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168756) roll requested 2024-11-13T14:29:28,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168763 2024-11-13T14:29:28,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168763 2024-11-13T14:29:28,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168763) roll requested 2024-11-13T14:29:28,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168771 2024-11-13T14:29:28,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168771 2024-11-13T14:29:28,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168771) roll requested 2024-11-13T14:29:28,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168779 2024-11-13T14:29:28,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168779 2024-11-13T14:29:28,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168779) roll requested 2024-11-13T14:29:28,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168788 2024-11-13T14:29:28,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168788 2024-11-13T14:29:28,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168788) roll requested 2024-11-13T14:29:28,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168794 2024-11-13T14:29:28,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168794 2024-11-13T14:29:28,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168794) roll requested 2024-11-13T14:29:28,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168800 2024-11-13T14:29:28,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168800 2024-11-13T14:29:28,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168800) roll requested 2024-11-13T14:29:28,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168808 2024-11-13T14:29:28,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168808 2024-11-13T14:29:28,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168808) roll requested 2024-11-13T14:29:28,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168815 2024-11-13T14:29:28,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168815 2024-11-13T14:29:28,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168815) roll requested 2024-11-13T14:29:28,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168824 2024-11-13T14:29:28,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168824 2024-11-13T14:29:28,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168824) roll requested 2024-11-13T14:29:28,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168832 2024-11-13T14:29:28,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168832 2024-11-13T14:29:28,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168832) roll requested 2024-11-13T14:29:28,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168839 2024-11-13T14:29:28,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168839 2024-11-13T14:29:28,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168839) roll requested 2024-11-13T14:29:28,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168847 2024-11-13T14:29:28,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168847 2024-11-13T14:29:28,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168847) roll requested 2024-11-13T14:29:28,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168855 2024-11-13T14:29:28,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168855 2024-11-13T14:29:28,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168855) roll requested 2024-11-13T14:29:28,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168862 2024-11-13T14:29:28,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168862 2024-11-13T14:29:28,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168862) roll requested 2024-11-13T14:29:28,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168870 2024-11-13T14:29:28,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168870 2024-11-13T14:29:28,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168870) roll requested 2024-11-13T14:29:28,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168879 2024-11-13T14:29:28,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168879 2024-11-13T14:29:28,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168879) roll requested 2024-11-13T14:29:28,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168888 2024-11-13T14:29:28,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168888 2024-11-13T14:29:28,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168888) roll requested 2024-11-13T14:29:28,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168897 2024-11-13T14:29:28,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168897 2024-11-13T14:29:28,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168897) roll requested 2024-11-13T14:29:28,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168904 2024-11-13T14:29:28,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168904 2024-11-13T14:29:28,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168904) roll requested 2024-11-13T14:29:28,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168912 2024-11-13T14:29:28,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168912 2024-11-13T14:29:28,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168912) roll requested 2024-11-13T14:29:28,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168921 2024-11-13T14:29:28,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168921 2024-11-13T14:29:28,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168921) roll requested 2024-11-13T14:29:28,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168927 2024-11-13T14:29:28,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168927 2024-11-13T14:29:28,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168927) roll requested 2024-11-13T14:29:28,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168935 2024-11-13T14:29:28,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168935 2024-11-13T14:29:28,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168935) roll requested 2024-11-13T14:29:28,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168943 2024-11-13T14:29:28,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168943 2024-11-13T14:29:28,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168943) roll requested 2024-11-13T14:29:28,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168954 2024-11-13T14:29:28,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168954 2024-11-13T14:29:28,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168954) roll requested 2024-11-13T14:29:28,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168963 2024-11-13T14:29:28,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168963 2024-11-13T14:29:28,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168963) roll requested 2024-11-13T14:29:28,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168971 2024-11-13T14:29:28,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168971 2024-11-13T14:29:28,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168971) roll requested 2024-11-13T14:29:28,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168980 2024-11-13T14:29:28,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168980 2024-11-13T14:29:28,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:28,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:28,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168980) roll requested 2024-11-13T14:29:28,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508168990 2024-11-13T14:29:28,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:28,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168990 2024-11-13T14:29:29,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508168990) roll requested 2024-11-13T14:29:29,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169000 2024-11-13T14:29:29,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508168990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169000 2024-11-13T14:29:29,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169000) roll requested 2024-11-13T14:29:29,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169007 2024-11-13T14:29:29,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169007 2024-11-13T14:29:29,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169007) roll requested 2024-11-13T14:29:29,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169012 2024-11-13T14:29:29,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169012 2024-11-13T14:29:29,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169012) roll requested 2024-11-13T14:29:29,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169017 2024-11-13T14:29:29,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169017 2024-11-13T14:29:29,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169017) roll requested 2024-11-13T14:29:29,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169024 2024-11-13T14:29:29,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169024 2024-11-13T14:29:29,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169024) roll requested 2024-11-13T14:29:29,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169031 2024-11-13T14:29:29,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169031 2024-11-13T14:29:29,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169031) roll requested 2024-11-13T14:29:29,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169038 2024-11-13T14:29:29,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169038 2024-11-13T14:29:29,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169038) roll requested 2024-11-13T14:29:29,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169048 2024-11-13T14:29:29,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169048 2024-11-13T14:29:29,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169048) roll requested 2024-11-13T14:29:29,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169056 2024-11-13T14:29:29,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169056 2024-11-13T14:29:29,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169056) roll requested 2024-11-13T14:29:29,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169063 2024-11-13T14:29:29,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169063 2024-11-13T14:29:29,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169063) roll requested 2024-11-13T14:29:29,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169068 2024-11-13T14:29:29,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169068 2024-11-13T14:29:29,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169068) roll requested 2024-11-13T14:29:29,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169075 2024-11-13T14:29:29,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169075 2024-11-13T14:29:29,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169075) roll requested 2024-11-13T14:29:29,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169083 2024-11-13T14:29:29,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169083 2024-11-13T14:29:29,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169083) roll requested 2024-11-13T14:29:29,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169090 2024-11-13T14:29:29,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169090 2024-11-13T14:29:29,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169090) roll requested 2024-11-13T14:29:29,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169099 2024-11-13T14:29:29,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169099 2024-11-13T14:29:29,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169099) roll requested 2024-11-13T14:29:29,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169109 2024-11-13T14:29:29,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169109 2024-11-13T14:29:29,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169109) roll requested 2024-11-13T14:29:29,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169119 2024-11-13T14:29:29,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169119 2024-11-13T14:29:29,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169119) roll requested 2024-11-13T14:29:29,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169126 2024-11-13T14:29:29,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169126 2024-11-13T14:29:29,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169126) roll requested 2024-11-13T14:29:29,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169136 2024-11-13T14:29:29,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169136 2024-11-13T14:29:29,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169136) roll requested 2024-11-13T14:29:29,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169144 2024-11-13T14:29:29,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169144 2024-11-13T14:29:29,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169144) roll requested 2024-11-13T14:29:29,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169151 2024-11-13T14:29:29,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169151 2024-11-13T14:29:29,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169151) roll requested 2024-11-13T14:29:29,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169159 2024-11-13T14:29:29,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169159 2024-11-13T14:29:29,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169159) roll requested 2024-11-13T14:29:29,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169167 2024-11-13T14:29:29,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169167 2024-11-13T14:29:29,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169167) roll requested 2024-11-13T14:29:29,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169175 2024-11-13T14:29:29,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169175 2024-11-13T14:29:29,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169175) roll requested 2024-11-13T14:29:29,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169183 2024-11-13T14:29:29,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169183 2024-11-13T14:29:29,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169183) roll requested 2024-11-13T14:29:29,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169191 2024-11-13T14:29:29,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169191 2024-11-13T14:29:29,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169191) roll requested 2024-11-13T14:29:29,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169203 2024-11-13T14:29:29,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169203 2024-11-13T14:29:29,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169203) roll requested 2024-11-13T14:29:29,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169211 2024-11-13T14:29:29,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169211 2024-11-13T14:29:29,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169211) roll requested 2024-11-13T14:29:29,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169221 2024-11-13T14:29:29,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169221 2024-11-13T14:29:29,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169221) roll requested 2024-11-13T14:29:29,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169228 2024-11-13T14:29:29,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169228 2024-11-13T14:29:29,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169228) roll requested 2024-11-13T14:29:29,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169236 2024-11-13T14:29:29,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169236 2024-11-13T14:29:29,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169236) roll requested 2024-11-13T14:29:29,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169242 2024-11-13T14:29:29,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169242 2024-11-13T14:29:29,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169242) roll requested 2024-11-13T14:29:29,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169254 2024-11-13T14:29:29,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169254 2024-11-13T14:29:29,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169254) roll requested 2024-11-13T14:29:29,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169260 2024-11-13T14:29:29,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169260 2024-11-13T14:29:29,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169260) roll requested 2024-11-13T14:29:29,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169265 2024-11-13T14:29:29,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169265 2024-11-13T14:29:29,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169265) roll requested 2024-11-13T14:29:29,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169270 2024-11-13T14:29:29,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169270 2024-11-13T14:29:29,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169270) roll requested 2024-11-13T14:29:29,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169275 2024-11-13T14:29:29,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169275 2024-11-13T14:29:29,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169275) roll requested 2024-11-13T14:29:29,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169282 2024-11-13T14:29:29,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169282 2024-11-13T14:29:29,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169282) roll requested 2024-11-13T14:29:29,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169290 2024-11-13T14:29:29,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169290 2024-11-13T14:29:29,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169290) roll requested 2024-11-13T14:29:29,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169298 2024-11-13T14:29:29,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169298 2024-11-13T14:29:29,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169298) roll requested 2024-11-13T14:29:29,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169305 2024-11-13T14:29:29,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169305 2024-11-13T14:29:29,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169305) roll requested 2024-11-13T14:29:29,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169313 2024-11-13T14:29:29,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169313 2024-11-13T14:29:29,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169313) roll requested 2024-11-13T14:29:29,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169323 2024-11-13T14:29:29,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169323 2024-11-13T14:29:29,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169323) roll requested 2024-11-13T14:29:29,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169331 2024-11-13T14:29:29,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169331 2024-11-13T14:29:29,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169331) roll requested 2024-11-13T14:29:29,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169341 2024-11-13T14:29:29,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169341 2024-11-13T14:29:29,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169341) roll requested 2024-11-13T14:29:29,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169349 2024-11-13T14:29:29,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169349 2024-11-13T14:29:29,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169349) roll requested 2024-11-13T14:29:29,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169357 2024-11-13T14:29:29,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169357 2024-11-13T14:29:29,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169357) roll requested 2024-11-13T14:29:29,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169366 2024-11-13T14:29:29,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169366 2024-11-13T14:29:29,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169366) roll requested 2024-11-13T14:29:29,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169375 2024-11-13T14:29:29,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169375 2024-11-13T14:29:29,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169375) roll requested 2024-11-13T14:29:29,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169383 2024-11-13T14:29:29,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169383 2024-11-13T14:29:29,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169383) roll requested 2024-11-13T14:29:29,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169393 2024-11-13T14:29:29,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169393 2024-11-13T14:29:29,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169393) roll requested 2024-11-13T14:29:29,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169404 2024-11-13T14:29:29,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169404 2024-11-13T14:29:29,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169404) roll requested 2024-11-13T14:29:29,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169410 2024-11-13T14:29:29,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169410 2024-11-13T14:29:29,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169410) roll requested 2024-11-13T14:29:29,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169415 2024-11-13T14:29:29,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169415 2024-11-13T14:29:29,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169415) roll requested 2024-11-13T14:29:29,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169422 2024-11-13T14:29:29,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169422 2024-11-13T14:29:29,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169422) roll requested 2024-11-13T14:29:29,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169428 2024-11-13T14:29:29,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169428 2024-11-13T14:29:29,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169428) roll requested 2024-11-13T14:29:29,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169435 2024-11-13T14:29:29,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169435 2024-11-13T14:29:29,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169435) roll requested 2024-11-13T14:29:29,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169443 2024-11-13T14:29:29,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169443 2024-11-13T14:29:29,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169443) roll requested 2024-11-13T14:29:29,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169451 2024-11-13T14:29:29,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169451 2024-11-13T14:29:29,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169451) roll requested 2024-11-13T14:29:29,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169464 2024-11-13T14:29:29,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169464 2024-11-13T14:29:29,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169464) roll requested 2024-11-13T14:29:29,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169474 2024-11-13T14:29:29,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169474 2024-11-13T14:29:29,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169474) roll requested 2024-11-13T14:29:29,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169479 2024-11-13T14:29:29,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169479 2024-11-13T14:29:29,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169479) roll requested 2024-11-13T14:29:29,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169485 2024-11-13T14:29:29,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169485 2024-11-13T14:29:29,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169485) roll requested 2024-11-13T14:29:29,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169492 2024-11-13T14:29:29,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169492 2024-11-13T14:29:29,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169492) roll requested 2024-11-13T14:29:29,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169500 2024-11-13T14:29:29,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169500 2024-11-13T14:29:29,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169500) roll requested 2024-11-13T14:29:29,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169506 2024-11-13T14:29:29,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169506 2024-11-13T14:29:29,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169506) roll requested 2024-11-13T14:29:29,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169513 2024-11-13T14:29:29,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169513 2024-11-13T14:29:29,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169513) roll requested 2024-11-13T14:29:29,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169518 2024-11-13T14:29:29,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169518 2024-11-13T14:29:29,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169518) roll requested 2024-11-13T14:29:29,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169523 2024-11-13T14:29:29,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169523 2024-11-13T14:29:29,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169523) roll requested 2024-11-13T14:29:29,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169529 2024-11-13T14:29:29,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169529 2024-11-13T14:29:29,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169529) roll requested 2024-11-13T14:29:29,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169538 2024-11-13T14:29:29,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169538 2024-11-13T14:29:29,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169538) roll requested 2024-11-13T14:29:29,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169544 2024-11-13T14:29:29,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169544 2024-11-13T14:29:29,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169544) roll requested 2024-11-13T14:29:29,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169550 2024-11-13T14:29:29,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169550 2024-11-13T14:29:29,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169550) roll requested 2024-11-13T14:29:29,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169555 2024-11-13T14:29:29,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169555 2024-11-13T14:29:29,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169555) roll requested 2024-11-13T14:29:29,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169561 2024-11-13T14:29:29,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169561 2024-11-13T14:29:29,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169561) roll requested 2024-11-13T14:29:29,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169566 2024-11-13T14:29:29,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169566 2024-11-13T14:29:29,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169566) roll requested 2024-11-13T14:29:29,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169571 2024-11-13T14:29:29,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169571 2024-11-13T14:29:29,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169571) roll requested 2024-11-13T14:29:29,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169576 2024-11-13T14:29:29,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169576 2024-11-13T14:29:29,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169576) roll requested 2024-11-13T14:29:29,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169583 2024-11-13T14:29:29,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169583 2024-11-13T14:29:29,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169583) roll requested 2024-11-13T14:29:29,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169590 2024-11-13T14:29:29,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169590 2024-11-13T14:29:29,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169590) roll requested 2024-11-13T14:29:29,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169599 2024-11-13T14:29:29,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169599 2024-11-13T14:29:29,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169599) roll requested 2024-11-13T14:29:29,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169606 2024-11-13T14:29:29,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169606 2024-11-13T14:29:29,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169606) roll requested 2024-11-13T14:29:29,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169615 2024-11-13T14:29:29,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169615 2024-11-13T14:29:29,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169615) roll requested 2024-11-13T14:29:29,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169625 2024-11-13T14:29:29,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169625 2024-11-13T14:29:29,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169625) roll requested 2024-11-13T14:29:29,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169632 2024-11-13T14:29:29,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169632 2024-11-13T14:29:29,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169632) roll requested 2024-11-13T14:29:29,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169644 2024-11-13T14:29:29,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169644 2024-11-13T14:29:29,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169644) roll requested 2024-11-13T14:29:29,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169654 2024-11-13T14:29:29,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169654 2024-11-13T14:29:29,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169654) roll requested 2024-11-13T14:29:29,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169662 2024-11-13T14:29:29,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169662 2024-11-13T14:29:29,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169662) roll requested 2024-11-13T14:29:29,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169667 2024-11-13T14:29:29,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169667 2024-11-13T14:29:29,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169667) roll requested 2024-11-13T14:29:29,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169679 2024-11-13T14:29:29,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169679 2024-11-13T14:29:29,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169679) roll requested 2024-11-13T14:29:29,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169685 2024-11-13T14:29:29,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169685 2024-11-13T14:29:29,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169685) roll requested 2024-11-13T14:29:29,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169690 2024-11-13T14:29:29,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169690 2024-11-13T14:29:29,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169690) roll requested 2024-11-13T14:29:29,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169697 2024-11-13T14:29:29,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169697 2024-11-13T14:29:29,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169697) roll requested 2024-11-13T14:29:29,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169705 2024-11-13T14:29:29,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169705 2024-11-13T14:29:29,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169705) roll requested 2024-11-13T14:29:29,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169713 2024-11-13T14:29:29,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169713 2024-11-13T14:29:29,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169713) roll requested 2024-11-13T14:29:29,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169720 2024-11-13T14:29:29,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169720 2024-11-13T14:29:29,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169720) roll requested 2024-11-13T14:29:29,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169728 2024-11-13T14:29:29,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169728 2024-11-13T14:29:29,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169728) roll requested 2024-11-13T14:29:29,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169736 2024-11-13T14:29:29,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169736 2024-11-13T14:29:29,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169736) roll requested 2024-11-13T14:29:29,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169746 2024-11-13T14:29:29,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169746 2024-11-13T14:29:29,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169746) roll requested 2024-11-13T14:29:29,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169753 2024-11-13T14:29:29,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169753 2024-11-13T14:29:29,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169753) roll requested 2024-11-13T14:29:29,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169760 2024-11-13T14:29:29,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169760 2024-11-13T14:29:29,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169760) roll requested 2024-11-13T14:29:29,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169767 2024-11-13T14:29:29,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169767 2024-11-13T14:29:29,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169767) roll requested 2024-11-13T14:29:29,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169772 2024-11-13T14:29:29,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169772 2024-11-13T14:29:29,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169772) roll requested 2024-11-13T14:29:29,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169778 2024-11-13T14:29:29,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169778 2024-11-13T14:29:29,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169778) roll requested 2024-11-13T14:29:29,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169784 2024-11-13T14:29:29,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169784 2024-11-13T14:29:29,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169784) roll requested 2024-11-13T14:29:29,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169792 2024-11-13T14:29:29,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169792 2024-11-13T14:29:29,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169792) roll requested 2024-11-13T14:29:29,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169800 2024-11-13T14:29:29,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169800 2024-11-13T14:29:29,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169800) roll requested 2024-11-13T14:29:29,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169805 2024-11-13T14:29:29,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169805 2024-11-13T14:29:29,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169805) roll requested 2024-11-13T14:29:29,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169811 2024-11-13T14:29:29,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169811 2024-11-13T14:29:29,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169811) roll requested 2024-11-13T14:29:29,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169818 2024-11-13T14:29:29,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169818 2024-11-13T14:29:29,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169818) roll requested 2024-11-13T14:29:29,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169830 2024-11-13T14:29:29,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169830 2024-11-13T14:29:29,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169830) roll requested 2024-11-13T14:29:29,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169840 2024-11-13T14:29:29,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169840 2024-11-13T14:29:29,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169840) roll requested 2024-11-13T14:29:29,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169852 2024-11-13T14:29:29,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169852 2024-11-13T14:29:29,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169852) roll requested 2024-11-13T14:29:29,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169858 2024-11-13T14:29:29,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169858 2024-11-13T14:29:29,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169858) roll requested 2024-11-13T14:29:29,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169865 2024-11-13T14:29:29,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169865 2024-11-13T14:29:29,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169865) roll requested 2024-11-13T14:29:29,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169870 2024-11-13T14:29:29,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169870 2024-11-13T14:29:29,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169870) roll requested 2024-11-13T14:29:29,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169876 2024-11-13T14:29:29,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169876 2024-11-13T14:29:29,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169876) roll requested 2024-11-13T14:29:29,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169883 2024-11-13T14:29:29,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169883 2024-11-13T14:29:29,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169883) roll requested 2024-11-13T14:29:29,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169894 2024-11-13T14:29:29,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169894 2024-11-13T14:29:29,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169894) roll requested 2024-11-13T14:29:29,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169901 2024-11-13T14:29:29,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169901 2024-11-13T14:29:29,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169901) roll requested 2024-11-13T14:29:29,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169908 2024-11-13T14:29:29,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169908 2024-11-13T14:29:29,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169908) roll requested 2024-11-13T14:29:29,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169915 2024-11-13T14:29:29,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169915 2024-11-13T14:29:29,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169915) roll requested 2024-11-13T14:29:29,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169922 2024-11-13T14:29:29,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169922 2024-11-13T14:29:29,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169922) roll requested 2024-11-13T14:29:29,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169932 2024-11-13T14:29:29,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169932 2024-11-13T14:29:29,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169932) roll requested 2024-11-13T14:29:29,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169940 2024-11-13T14:29:29,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169940 2024-11-13T14:29:29,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169940) roll requested 2024-11-13T14:29:29,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169949 2024-11-13T14:29:29,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169949 2024-11-13T14:29:29,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169949) roll requested 2024-11-13T14:29:29,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169958 2024-11-13T14:29:29,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169958 2024-11-13T14:29:29,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169958) roll requested 2024-11-13T14:29:29,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169968 2024-11-13T14:29:29,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169968 2024-11-13T14:29:29,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169968) roll requested 2024-11-13T14:29:29,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169979 2024-11-13T14:29:29,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169979 2024-11-13T14:29:29,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169979) roll requested 2024-11-13T14:29:29,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169987 2024-11-13T14:29:29,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:29,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169987 2024-11-13T14:29:29,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:29,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:29,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169987) roll requested 2024-11-13T14:29:29,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508169997 2024-11-13T14:29:30,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169997 2024-11-13T14:29:30,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508169997) roll requested 2024-11-13T14:29:30,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170006 2024-11-13T14:29:30,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508169997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170006 2024-11-13T14:29:30,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170006) roll requested 2024-11-13T14:29:30,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170011 2024-11-13T14:29:30,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170011 2024-11-13T14:29:30,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170011) roll requested 2024-11-13T14:29:30,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170018 2024-11-13T14:29:30,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170018 2024-11-13T14:29:30,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170018) roll requested 2024-11-13T14:29:30,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170026 2024-11-13T14:29:30,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170026 2024-11-13T14:29:30,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170026) roll requested 2024-11-13T14:29:30,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170033 2024-11-13T14:29:30,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170033 2024-11-13T14:29:30,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170033) roll requested 2024-11-13T14:29:30,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170041 2024-11-13T14:29:30,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170041 2024-11-13T14:29:30,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170041) roll requested 2024-11-13T14:29:30,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170050 2024-11-13T14:29:30,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170050 2024-11-13T14:29:30,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170050) roll requested 2024-11-13T14:29:30,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170058 2024-11-13T14:29:30,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170058 2024-11-13T14:29:30,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170058) roll requested 2024-11-13T14:29:30,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170065 2024-11-13T14:29:30,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170065 2024-11-13T14:29:30,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170065) roll requested 2024-11-13T14:29:30,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170073 2024-11-13T14:29:30,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170073 2024-11-13T14:29:30,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170073) roll requested 2024-11-13T14:29:30,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170081 2024-11-13T14:29:30,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170081 2024-11-13T14:29:30,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170081) roll requested 2024-11-13T14:29:30,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170091 2024-11-13T14:29:30,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170091 2024-11-13T14:29:30,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170091) roll requested 2024-11-13T14:29:30,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170100 2024-11-13T14:29:30,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170100 2024-11-13T14:29:30,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170100) roll requested 2024-11-13T14:29:30,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170107 2024-11-13T14:29:30,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170107 2024-11-13T14:29:30,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170107) roll requested 2024-11-13T14:29:30,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170116 2024-11-13T14:29:30,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170116 2024-11-13T14:29:30,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170116) roll requested 2024-11-13T14:29:30,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170124 2024-11-13T14:29:30,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170124 2024-11-13T14:29:30,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170124) roll requested 2024-11-13T14:29:30,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170132 2024-11-13T14:29:30,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170132 2024-11-13T14:29:30,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170132) roll requested 2024-11-13T14:29:30,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170142 2024-11-13T14:29:30,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170142 2024-11-13T14:29:30,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170142) roll requested 2024-11-13T14:29:30,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170152 2024-11-13T14:29:30,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170152 2024-11-13T14:29:30,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170152) roll requested 2024-11-13T14:29:30,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170160 2024-11-13T14:29:30,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170160 2024-11-13T14:29:30,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170160) roll requested 2024-11-13T14:29:30,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170170 2024-11-13T14:29:30,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170170 2024-11-13T14:29:30,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170170) roll requested 2024-11-13T14:29:30,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170180 2024-11-13T14:29:30,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170180 2024-11-13T14:29:30,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170180) roll requested 2024-11-13T14:29:30,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170189 2024-11-13T14:29:30,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170189 2024-11-13T14:29:30,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170189) roll requested 2024-11-13T14:29:30,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170197 2024-11-13T14:29:30,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170197 2024-11-13T14:29:30,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170197) roll requested 2024-11-13T14:29:30,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170206 2024-11-13T14:29:30,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170206 2024-11-13T14:29:30,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170206) roll requested 2024-11-13T14:29:30,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170215 2024-11-13T14:29:30,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170215 2024-11-13T14:29:30,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170215) roll requested 2024-11-13T14:29:30,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170224 2024-11-13T14:29:30,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170224 2024-11-13T14:29:30,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170224) roll requested 2024-11-13T14:29:30,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170233 2024-11-13T14:29:30,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170233 2024-11-13T14:29:30,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170233) roll requested 2024-11-13T14:29:30,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170240 2024-11-13T14:29:30,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170240 2024-11-13T14:29:30,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170240) roll requested 2024-11-13T14:29:30,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170247 2024-11-13T14:29:30,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170247 2024-11-13T14:29:30,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170247) roll requested 2024-11-13T14:29:30,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170255 2024-11-13T14:29:30,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170255 2024-11-13T14:29:30,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170255) roll requested 2024-11-13T14:29:30,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170263 2024-11-13T14:29:30,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170263 2024-11-13T14:29:30,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170263) roll requested 2024-11-13T14:29:30,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170272 2024-11-13T14:29:30,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170272 2024-11-13T14:29:30,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170272) roll requested 2024-11-13T14:29:30,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170281 2024-11-13T14:29:30,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170281 2024-11-13T14:29:30,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170281) roll requested 2024-11-13T14:29:30,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170290 2024-11-13T14:29:30,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170290 2024-11-13T14:29:30,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170290) roll requested 2024-11-13T14:29:30,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170299 2024-11-13T14:29:30,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170299 2024-11-13T14:29:30,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170299) roll requested 2024-11-13T14:29:30,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170308 2024-11-13T14:29:30,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170308 2024-11-13T14:29:30,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170308) roll requested 2024-11-13T14:29:30,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170315 2024-11-13T14:29:30,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170315 2024-11-13T14:29:30,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170315) roll requested 2024-11-13T14:29:30,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170325 2024-11-13T14:29:30,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170325 2024-11-13T14:29:30,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170325) roll requested 2024-11-13T14:29:30,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170332 2024-11-13T14:29:30,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170332 2024-11-13T14:29:30,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170332) roll requested 2024-11-13T14:29:30,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170339 2024-11-13T14:29:30,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170339 2024-11-13T14:29:30,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170339) roll requested 2024-11-13T14:29:30,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170348 2024-11-13T14:29:30,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170348 2024-11-13T14:29:30,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170348) roll requested 2024-11-13T14:29:30,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170357 2024-11-13T14:29:30,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170357 2024-11-13T14:29:30,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170357) roll requested 2024-11-13T14:29:30,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170366 2024-11-13T14:29:30,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170366 2024-11-13T14:29:30,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170366) roll requested 2024-11-13T14:29:30,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170378 2024-11-13T14:29:30,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170378 2024-11-13T14:29:30,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170378) roll requested 2024-11-13T14:29:30,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170387 2024-11-13T14:29:30,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170387 2024-11-13T14:29:30,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170387) roll requested 2024-11-13T14:29:30,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170395 2024-11-13T14:29:30,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170395 2024-11-13T14:29:30,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170395) roll requested 2024-11-13T14:29:30,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170402 2024-11-13T14:29:30,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170402 2024-11-13T14:29:30,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170402) roll requested 2024-11-13T14:29:30,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170411 2024-11-13T14:29:30,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170411 2024-11-13T14:29:30,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170411) roll requested 2024-11-13T14:29:30,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170421 2024-11-13T14:29:30,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170421 2024-11-13T14:29:30,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170421) roll requested 2024-11-13T14:29:30,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170430 2024-11-13T14:29:30,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170430 2024-11-13T14:29:30,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170430) roll requested 2024-11-13T14:29:30,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170438 2024-11-13T14:29:30,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170438 2024-11-13T14:29:30,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170438) roll requested 2024-11-13T14:29:30,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170452 2024-11-13T14:29:30,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170452 2024-11-13T14:29:30,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170452) roll requested 2024-11-13T14:29:30,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170457 2024-11-13T14:29:30,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170457 2024-11-13T14:29:30,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170457) roll requested 2024-11-13T14:29:30,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170464 2024-11-13T14:29:30,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170464 2024-11-13T14:29:30,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170464) roll requested 2024-11-13T14:29:30,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170474 2024-11-13T14:29:30,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170474 2024-11-13T14:29:30,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170474) roll requested 2024-11-13T14:29:30,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170480 2024-11-13T14:29:30,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170480 2024-11-13T14:29:30,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170480) roll requested 2024-11-13T14:29:30,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170485 2024-11-13T14:29:30,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170485 2024-11-13T14:29:30,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170485) roll requested 2024-11-13T14:29:30,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170490 2024-11-13T14:29:30,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170490 2024-11-13T14:29:30,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170490) roll requested 2024-11-13T14:29:30,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170495 2024-11-13T14:29:30,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170495 2024-11-13T14:29:30,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170495) roll requested 2024-11-13T14:29:30,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170501 2024-11-13T14:29:30,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170501 2024-11-13T14:29:30,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170501) roll requested 2024-11-13T14:29:30,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170506 2024-11-13T14:29:30,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170506 2024-11-13T14:29:30,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170506) roll requested 2024-11-13T14:29:30,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170513 2024-11-13T14:29:30,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170513 2024-11-13T14:29:30,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170513) roll requested 2024-11-13T14:29:30,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170521 2024-11-13T14:29:30,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170521 2024-11-13T14:29:30,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170521) roll requested 2024-11-13T14:29:30,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170527 2024-11-13T14:29:30,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170527 2024-11-13T14:29:30,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170527) roll requested 2024-11-13T14:29:30,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170532 2024-11-13T14:29:30,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170532 2024-11-13T14:29:30,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170532) roll requested 2024-11-13T14:29:30,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170540 2024-11-13T14:29:30,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170540 2024-11-13T14:29:30,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170540) roll requested 2024-11-13T14:29:30,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170547 2024-11-13T14:29:30,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170547 2024-11-13T14:29:30,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170547) roll requested 2024-11-13T14:29:30,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170554 2024-11-13T14:29:30,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170554 2024-11-13T14:29:30,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170554) roll requested 2024-11-13T14:29:30,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170563 2024-11-13T14:29:30,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170563 2024-11-13T14:29:30,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170563) roll requested 2024-11-13T14:29:30,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170572 2024-11-13T14:29:30,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170572 2024-11-13T14:29:30,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170572) roll requested 2024-11-13T14:29:30,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170581 2024-11-13T14:29:30,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170581 2024-11-13T14:29:30,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170581) roll requested 2024-11-13T14:29:30,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170593 2024-11-13T14:29:30,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170593 2024-11-13T14:29:30,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170593) roll requested 2024-11-13T14:29:30,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170601 2024-11-13T14:29:30,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170601 2024-11-13T14:29:30,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170601) roll requested 2024-11-13T14:29:30,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170610 2024-11-13T14:29:30,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170610 2024-11-13T14:29:30,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170610) roll requested 2024-11-13T14:29:30,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170617 2024-11-13T14:29:30,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170617 2024-11-13T14:29:30,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170617) roll requested 2024-11-13T14:29:30,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170625 2024-11-13T14:29:30,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170625 2024-11-13T14:29:30,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170625) roll requested 2024-11-13T14:29:30,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170635 2024-11-13T14:29:30,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170635 2024-11-13T14:29:30,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170635) roll requested 2024-11-13T14:29:30,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170644 2024-11-13T14:29:30,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170644 2024-11-13T14:29:30,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170644) roll requested 2024-11-13T14:29:30,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170655 2024-11-13T14:29:30,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170655 2024-11-13T14:29:30,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170655) roll requested 2024-11-13T14:29:30,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170662 2024-11-13T14:29:30,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170662 2024-11-13T14:29:30,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170662) roll requested 2024-11-13T14:29:30,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170670 2024-11-13T14:29:30,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170670 2024-11-13T14:29:30,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170670) roll requested 2024-11-13T14:29:30,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170743 2024-11-13T14:29:30,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170743 2024-11-13T14:29:30,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170743) roll requested 2024-11-13T14:29:30,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170750 2024-11-13T14:29:30,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170750 2024-11-13T14:29:30,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170750) roll requested 2024-11-13T14:29:30,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170759 2024-11-13T14:29:30,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170759 2024-11-13T14:29:30,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170759) roll requested 2024-11-13T14:29:30,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170764 2024-11-13T14:29:30,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170764 2024-11-13T14:29:30,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170764) roll requested 2024-11-13T14:29:30,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170770 2024-11-13T14:29:30,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170770 2024-11-13T14:29:30,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170770) roll requested 2024-11-13T14:29:30,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170777 2024-11-13T14:29:30,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170777 2024-11-13T14:29:30,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170777) roll requested 2024-11-13T14:29:30,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170785 2024-11-13T14:29:30,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170785 2024-11-13T14:29:30,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170785) roll requested 2024-11-13T14:29:30,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170793 2024-11-13T14:29:30,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170793 2024-11-13T14:29:30,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170793) roll requested 2024-11-13T14:29:30,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170801 2024-11-13T14:29:30,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170801 2024-11-13T14:29:30,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170801) roll requested 2024-11-13T14:29:30,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170810 2024-11-13T14:29:30,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170810 2024-11-13T14:29:30,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170810) roll requested 2024-11-13T14:29:30,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170818 2024-11-13T14:29:30,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170818 2024-11-13T14:29:30,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170818) roll requested 2024-11-13T14:29:30,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170826 2024-11-13T14:29:30,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170826 2024-11-13T14:29:30,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170826) roll requested 2024-11-13T14:29:30,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170833 2024-11-13T14:29:30,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170833 2024-11-13T14:29:30,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170833) roll requested 2024-11-13T14:29:30,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170840 2024-11-13T14:29:30,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170840 2024-11-13T14:29:30,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170840) roll requested 2024-11-13T14:29:30,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170849 2024-11-13T14:29:30,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170849 2024-11-13T14:29:30,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170849) roll requested 2024-11-13T14:29:30,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170857 2024-11-13T14:29:30,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170857 2024-11-13T14:29:30,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170857) roll requested 2024-11-13T14:29:30,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170866 2024-11-13T14:29:30,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170866 2024-11-13T14:29:30,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170866) roll requested 2024-11-13T14:29:30,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170876 2024-11-13T14:29:30,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170876 2024-11-13T14:29:30,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170876) roll requested 2024-11-13T14:29:30,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170888 2024-11-13T14:29:30,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170888 2024-11-13T14:29:30,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170888) roll requested 2024-11-13T14:29:30,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170895 2024-11-13T14:29:30,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170895 2024-11-13T14:29:30,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170895) roll requested 2024-11-13T14:29:30,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170908 2024-11-13T14:29:30,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170908 2024-11-13T14:29:30,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170908) roll requested 2024-11-13T14:29:30,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170916 2024-11-13T14:29:30,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170916 2024-11-13T14:29:30,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170916) roll requested 2024-11-13T14:29:30,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170922 2024-11-13T14:29:30,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170922 2024-11-13T14:29:30,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170922) roll requested 2024-11-13T14:29:30,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170931 2024-11-13T14:29:30,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170931 2024-11-13T14:29:30,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170931) roll requested 2024-11-13T14:29:30,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170936 2024-11-13T14:29:30,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170936 2024-11-13T14:29:30,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170936) roll requested 2024-11-13T14:29:30,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170942 2024-11-13T14:29:30,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170942 2024-11-13T14:29:30,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170942) roll requested 2024-11-13T14:29:30,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170947 2024-11-13T14:29:30,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170947 2024-11-13T14:29:30,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170947) roll requested 2024-11-13T14:29:30,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170952 2024-11-13T14:29:30,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170952 2024-11-13T14:29:30,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170952) roll requested 2024-11-13T14:29:30,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170957 2024-11-13T14:29:30,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170957 2024-11-13T14:29:30,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170957) roll requested 2024-11-13T14:29:30,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170965 2024-11-13T14:29:30,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170965 2024-11-13T14:29:30,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170965) roll requested 2024-11-13T14:29:30,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170970 2024-11-13T14:29:30,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170970 2024-11-13T14:29:30,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170970) roll requested 2024-11-13T14:29:30,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170976 2024-11-13T14:29:30,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170976 2024-11-13T14:29:30,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170976) roll requested 2024-11-13T14:29:30,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170981 2024-11-13T14:29:30,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170981 2024-11-13T14:29:30,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170981) roll requested 2024-11-13T14:29:30,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170987 2024-11-13T14:29:30,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170987 2024-11-13T14:29:30,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170987) roll requested 2024-11-13T14:29:30,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170992 2024-11-13T14:29:30,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:30,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170992 2024-11-13T14:29:30,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:30,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:30,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170992) roll requested 2024-11-13T14:29:30,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508170998 2024-11-13T14:29:31,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170998 2024-11-13T14:29:31,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508170998) roll requested 2024-11-13T14:29:31,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171004 2024-11-13T14:29:31,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508170998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171004 2024-11-13T14:29:31,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171004) roll requested 2024-11-13T14:29:31,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171010 2024-11-13T14:29:31,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171010 2024-11-13T14:29:31,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171010) roll requested 2024-11-13T14:29:31,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171017 2024-11-13T14:29:31,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171017 2024-11-13T14:29:31,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171017) roll requested 2024-11-13T14:29:31,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171024 2024-11-13T14:29:31,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171024 2024-11-13T14:29:31,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171024) roll requested 2024-11-13T14:29:31,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171030 2024-11-13T14:29:31,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171030 2024-11-13T14:29:31,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171030) roll requested 2024-11-13T14:29:31,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171035 2024-11-13T14:29:31,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171035 2024-11-13T14:29:31,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171035) roll requested 2024-11-13T14:29:31,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171039 2024-11-13T14:29:31,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171039 2024-11-13T14:29:31,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171039) roll requested 2024-11-13T14:29:31,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171047 2024-11-13T14:29:31,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171047 2024-11-13T14:29:31,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171047) roll requested 2024-11-13T14:29:31,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171055 2024-11-13T14:29:31,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171055 2024-11-13T14:29:31,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171055) roll requested 2024-11-13T14:29:31,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171060 2024-11-13T14:29:31,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171060 2024-11-13T14:29:31,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171060) roll requested 2024-11-13T14:29:31,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171065 2024-11-13T14:29:31,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171065 2024-11-13T14:29:31,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171065) roll requested 2024-11-13T14:29:31,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171070 2024-11-13T14:29:31,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171070 2024-11-13T14:29:31,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171070) roll requested 2024-11-13T14:29:31,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171079 2024-11-13T14:29:31,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171079 2024-11-13T14:29:31,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171079) roll requested 2024-11-13T14:29:31,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171085 2024-11-13T14:29:31,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171085 2024-11-13T14:29:31,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171085) roll requested 2024-11-13T14:29:31,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171090 2024-11-13T14:29:31,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171090 2024-11-13T14:29:31,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171090) roll requested 2024-11-13T14:29:31,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171096 2024-11-13T14:29:31,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171096 2024-11-13T14:29:31,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171096) roll requested 2024-11-13T14:29:31,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171102 2024-11-13T14:29:31,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171102 2024-11-13T14:29:31,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171102) roll requested 2024-11-13T14:29:31,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171108 2024-11-13T14:29:31,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171108 2024-11-13T14:29:31,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171108) roll requested 2024-11-13T14:29:31,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171115 2024-11-13T14:29:31,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171115 2024-11-13T14:29:31,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171115) roll requested 2024-11-13T14:29:31,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171121 2024-11-13T14:29:31,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171121 2024-11-13T14:29:31,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171121) roll requested 2024-11-13T14:29:31,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171129 2024-11-13T14:29:31,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171129 2024-11-13T14:29:31,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171129) roll requested 2024-11-13T14:29:31,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171137 2024-11-13T14:29:31,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171137 2024-11-13T14:29:31,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171137) roll requested 2024-11-13T14:29:31,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171144 2024-11-13T14:29:31,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171144 2024-11-13T14:29:31,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171144) roll requested 2024-11-13T14:29:31,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171153 2024-11-13T14:29:31,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171153 2024-11-13T14:29:31,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171153) roll requested 2024-11-13T14:29:31,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171160 2024-11-13T14:29:31,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171160 2024-11-13T14:29:31,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171160) roll requested 2024-11-13T14:29:31,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171167 2024-11-13T14:29:31,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171167 2024-11-13T14:29:31,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171167) roll requested 2024-11-13T14:29:31,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171175 2024-11-13T14:29:31,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171175 2024-11-13T14:29:31,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171175) roll requested 2024-11-13T14:29:31,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171183 2024-11-13T14:29:31,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171183 2024-11-13T14:29:31,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171183) roll requested 2024-11-13T14:29:31,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171193 2024-11-13T14:29:31,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171193 2024-11-13T14:29:31,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171193) roll requested 2024-11-13T14:29:31,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171203 2024-11-13T14:29:31,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171203 2024-11-13T14:29:31,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171203) roll requested 2024-11-13T14:29:31,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171210 2024-11-13T14:29:31,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171210 2024-11-13T14:29:31,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171210) roll requested 2024-11-13T14:29:31,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171217 2024-11-13T14:29:31,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171217 2024-11-13T14:29:31,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171217) roll requested 2024-11-13T14:29:31,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171225 2024-11-13T14:29:31,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171225 2024-11-13T14:29:31,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171225) roll requested 2024-11-13T14:29:31,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171232 2024-11-13T14:29:31,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171232 2024-11-13T14:29:31,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171232) roll requested 2024-11-13T14:29:31,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171240 2024-11-13T14:29:31,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171240 2024-11-13T14:29:31,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171240) roll requested 2024-11-13T14:29:31,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171248 2024-11-13T14:29:31,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171248 2024-11-13T14:29:31,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171248) roll requested 2024-11-13T14:29:31,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171255 2024-11-13T14:29:31,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171255 2024-11-13T14:29:31,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171255) roll requested 2024-11-13T14:29:31,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171264 2024-11-13T14:29:31,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171264 2024-11-13T14:29:31,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171264) roll requested 2024-11-13T14:29:31,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171272 2024-11-13T14:29:31,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171272 2024-11-13T14:29:31,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171272) roll requested 2024-11-13T14:29:31,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171281 2024-11-13T14:29:31,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171281 2024-11-13T14:29:31,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171281) roll requested 2024-11-13T14:29:31,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171288 2024-11-13T14:29:31,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171288 2024-11-13T14:29:31,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171288) roll requested 2024-11-13T14:29:31,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171296 2024-11-13T14:29:31,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171296 2024-11-13T14:29:31,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171296) roll requested 2024-11-13T14:29:31,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171304 2024-11-13T14:29:31,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171304 2024-11-13T14:29:31,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171304) roll requested 2024-11-13T14:29:31,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171312 2024-11-13T14:29:31,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171312 2024-11-13T14:29:31,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171312) roll requested 2024-11-13T14:29:31,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171318 2024-11-13T14:29:31,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171318 2024-11-13T14:29:31,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171318) roll requested 2024-11-13T14:29:31,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171324 2024-11-13T14:29:31,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171324 2024-11-13T14:29:31,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171324) roll requested 2024-11-13T14:29:31,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171332 2024-11-13T14:29:31,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171332 2024-11-13T14:29:31,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171332) roll requested 2024-11-13T14:29:31,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171340 2024-11-13T14:29:31,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171340 2024-11-13T14:29:31,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171340) roll requested 2024-11-13T14:29:31,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171347 2024-11-13T14:29:31,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171347 2024-11-13T14:29:31,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171347) roll requested 2024-11-13T14:29:31,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171355 2024-11-13T14:29:31,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171355 2024-11-13T14:29:31,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171355) roll requested 2024-11-13T14:29:31,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171361 2024-11-13T14:29:31,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171361 2024-11-13T14:29:31,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171361) roll requested 2024-11-13T14:29:31,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171367 2024-11-13T14:29:31,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171367 2024-11-13T14:29:31,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171367) roll requested 2024-11-13T14:29:31,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171374 2024-11-13T14:29:31,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171374 2024-11-13T14:29:31,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171374) roll requested 2024-11-13T14:29:31,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171379 2024-11-13T14:29:31,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171379 2024-11-13T14:29:31,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171379) roll requested 2024-11-13T14:29:31,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171384 2024-11-13T14:29:31,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171384 2024-11-13T14:29:31,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171384) roll requested 2024-11-13T14:29:31,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171389 2024-11-13T14:29:31,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171389 2024-11-13T14:29:31,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171389) roll requested 2024-11-13T14:29:31,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171396 2024-11-13T14:29:31,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171396 2024-11-13T14:29:31,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171396) roll requested 2024-11-13T14:29:31,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171403 2024-11-13T14:29:31,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171403 2024-11-13T14:29:31,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171403) roll requested 2024-11-13T14:29:31,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171411 2024-11-13T14:29:31,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171411 2024-11-13T14:29:31,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171411) roll requested 2024-11-13T14:29:31,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171420 2024-11-13T14:29:31,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171420 2024-11-13T14:29:31,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171420) roll requested 2024-11-13T14:29:31,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171430 2024-11-13T14:29:31,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171430 2024-11-13T14:29:31,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171430) roll requested 2024-11-13T14:29:31,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171436 2024-11-13T14:29:31,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171436 2024-11-13T14:29:31,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171436) roll requested 2024-11-13T14:29:31,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171443 2024-11-13T14:29:31,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171443 2024-11-13T14:29:31,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171443) roll requested 2024-11-13T14:29:31,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171452 2024-11-13T14:29:31,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171452 2024-11-13T14:29:31,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171452) roll requested 2024-11-13T14:29:31,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171460 2024-11-13T14:29:31,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171460 2024-11-13T14:29:31,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171460) roll requested 2024-11-13T14:29:31,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171469 2024-11-13T14:29:31,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171469 2024-11-13T14:29:31,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171469) roll requested 2024-11-13T14:29:31,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171475 2024-11-13T14:29:31,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171475 2024-11-13T14:29:31,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171475) roll requested 2024-11-13T14:29:31,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171481 2024-11-13T14:29:31,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171481 2024-11-13T14:29:31,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171481) roll requested 2024-11-13T14:29:31,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171488 2024-11-13T14:29:31,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171488 2024-11-13T14:29:31,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171488) roll requested 2024-11-13T14:29:31,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171496 2024-11-13T14:29:31,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171496 2024-11-13T14:29:31,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171496) roll requested 2024-11-13T14:29:31,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171505 2024-11-13T14:29:31,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171505 2024-11-13T14:29:31,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171505) roll requested 2024-11-13T14:29:31,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171512 2024-11-13T14:29:31,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171512 2024-11-13T14:29:31,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171512) roll requested 2024-11-13T14:29:31,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171519 2024-11-13T14:29:31,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171519 2024-11-13T14:29:31,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171519) roll requested 2024-11-13T14:29:31,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171528 2024-11-13T14:29:31,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171528 2024-11-13T14:29:31,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171528) roll requested 2024-11-13T14:29:31,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171533 2024-11-13T14:29:31,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171533 2024-11-13T14:29:31,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171533) roll requested 2024-11-13T14:29:31,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171538 2024-11-13T14:29:31,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171538 2024-11-13T14:29:31,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171538) roll requested 2024-11-13T14:29:31,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171546 2024-11-13T14:29:31,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171546 2024-11-13T14:29:31,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171546) roll requested 2024-11-13T14:29:31,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171554 2024-11-13T14:29:31,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171554 2024-11-13T14:29:31,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171554) roll requested 2024-11-13T14:29:31,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171561 2024-11-13T14:29:31,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171561 2024-11-13T14:29:31,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171561) roll requested 2024-11-13T14:29:31,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171569 2024-11-13T14:29:31,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171569 2024-11-13T14:29:31,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171569) roll requested 2024-11-13T14:29:31,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171577 2024-11-13T14:29:31,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171577 2024-11-13T14:29:31,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171577) roll requested 2024-11-13T14:29:31,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171585 2024-11-13T14:29:31,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171585 2024-11-13T14:29:31,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171585) roll requested 2024-11-13T14:29:31,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171593 2024-11-13T14:29:31,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171593 2024-11-13T14:29:31,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171593) roll requested 2024-11-13T14:29:31,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171601 2024-11-13T14:29:31,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171601 2024-11-13T14:29:31,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171601) roll requested 2024-11-13T14:29:31,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171610 2024-11-13T14:29:31,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171610 2024-11-13T14:29:31,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171610) roll requested 2024-11-13T14:29:31,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171621 2024-11-13T14:29:31,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171621 2024-11-13T14:29:31,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171621) roll requested 2024-11-13T14:29:31,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171631 2024-11-13T14:29:31,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171631 2024-11-13T14:29:31,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171631) roll requested 2024-11-13T14:29:31,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171640 2024-11-13T14:29:31,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171640 2024-11-13T14:29:31,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171640) roll requested 2024-11-13T14:29:31,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171647 2024-11-13T14:29:31,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171647 2024-11-13T14:29:31,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171647) roll requested 2024-11-13T14:29:31,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171656 2024-11-13T14:29:31,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171656 2024-11-13T14:29:31,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171656) roll requested 2024-11-13T14:29:31,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171665 2024-11-13T14:29:31,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171665 2024-11-13T14:29:31,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171665) roll requested 2024-11-13T14:29:31,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171673 2024-11-13T14:29:31,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171673 2024-11-13T14:29:31,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171673) roll requested 2024-11-13T14:29:31,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171679 2024-11-13T14:29:31,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171679 2024-11-13T14:29:31,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171679) roll requested 2024-11-13T14:29:31,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171686 2024-11-13T14:29:31,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171686 2024-11-13T14:29:31,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171686) roll requested 2024-11-13T14:29:31,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171694 2024-11-13T14:29:31,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171694 2024-11-13T14:29:31,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171694) roll requested 2024-11-13T14:29:31,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171702 2024-11-13T14:29:31,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171702 2024-11-13T14:29:31,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171702) roll requested 2024-11-13T14:29:31,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171710 2024-11-13T14:29:31,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171710 2024-11-13T14:29:31,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171710) roll requested 2024-11-13T14:29:31,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171720 2024-11-13T14:29:31,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171720 2024-11-13T14:29:31,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171720) roll requested 2024-11-13T14:29:31,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171730 2024-11-13T14:29:31,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171730 2024-11-13T14:29:31,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171730) roll requested 2024-11-13T14:29:31,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171737 2024-11-13T14:29:31,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171737 2024-11-13T14:29:31,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171737) roll requested 2024-11-13T14:29:31,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171745 2024-11-13T14:29:31,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171745 2024-11-13T14:29:31,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171745) roll requested 2024-11-13T14:29:31,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171755 2024-11-13T14:29:31,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171755 2024-11-13T14:29:31,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171755) roll requested 2024-11-13T14:29:31,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171764 2024-11-13T14:29:31,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171764 2024-11-13T14:29:31,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171764) roll requested 2024-11-13T14:29:31,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171773 2024-11-13T14:29:31,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171773 2024-11-13T14:29:31,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171773) roll requested 2024-11-13T14:29:31,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171781 2024-11-13T14:29:31,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171781 2024-11-13T14:29:31,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171781) roll requested 2024-11-13T14:29:31,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171790 2024-11-13T14:29:31,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171790 2024-11-13T14:29:31,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171790) roll requested 2024-11-13T14:29:31,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171799 2024-11-13T14:29:31,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171799 2024-11-13T14:29:31,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171799) roll requested 2024-11-13T14:29:31,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171806 2024-11-13T14:29:31,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171806 2024-11-13T14:29:31,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171806) roll requested 2024-11-13T14:29:31,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171814 2024-11-13T14:29:31,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171814 2024-11-13T14:29:31,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171814) roll requested 2024-11-13T14:29:31,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171823 2024-11-13T14:29:31,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171823 2024-11-13T14:29:31,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171823) roll requested 2024-11-13T14:29:31,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171830 2024-11-13T14:29:31,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171830 2024-11-13T14:29:31,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171830) roll requested 2024-11-13T14:29:31,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171836 2024-11-13T14:29:31,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171836 2024-11-13T14:29:31,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171836) roll requested 2024-11-13T14:29:31,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171842 2024-11-13T14:29:31,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171842 2024-11-13T14:29:31,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171842) roll requested 2024-11-13T14:29:31,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171849 2024-11-13T14:29:31,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171849 2024-11-13T14:29:31,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171849) roll requested 2024-11-13T14:29:31,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171858 2024-11-13T14:29:31,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171858 2024-11-13T14:29:31,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171858) roll requested 2024-11-13T14:29:31,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171868 2024-11-13T14:29:31,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171868 2024-11-13T14:29:31,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171868) roll requested 2024-11-13T14:29:31,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171875 2024-11-13T14:29:31,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171875 2024-11-13T14:29:31,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171875) roll requested 2024-11-13T14:29:31,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171883 2024-11-13T14:29:31,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171883 2024-11-13T14:29:31,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171883) roll requested 2024-11-13T14:29:31,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171892 2024-11-13T14:29:31,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171892 2024-11-13T14:29:31,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171892) roll requested 2024-11-13T14:29:31,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171900 2024-11-13T14:29:31,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171900 2024-11-13T14:29:31,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171900) roll requested 2024-11-13T14:29:31,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171905 2024-11-13T14:29:31,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171905 2024-11-13T14:29:31,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171905) roll requested 2024-11-13T14:29:31,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171910 2024-11-13T14:29:31,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171910 2024-11-13T14:29:31,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171910) roll requested 2024-11-13T14:29:31,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171915 2024-11-13T14:29:31,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171915 2024-11-13T14:29:31,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171915) roll requested 2024-11-13T14:29:31,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171921 2024-11-13T14:29:31,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171921 2024-11-13T14:29:31,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171921) roll requested 2024-11-13T14:29:31,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171927 2024-11-13T14:29:31,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171927 2024-11-13T14:29:31,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171927) roll requested 2024-11-13T14:29:31,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171933 2024-11-13T14:29:31,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171933 2024-11-13T14:29:31,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171933) roll requested 2024-11-13T14:29:31,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171941 2024-11-13T14:29:31,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171941 2024-11-13T14:29:31,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171941) roll requested 2024-11-13T14:29:31,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171949 2024-11-13T14:29:31,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171949 2024-11-13T14:29:31,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171949) roll requested 2024-11-13T14:29:31,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171958 2024-11-13T14:29:31,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171958 2024-11-13T14:29:31,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171958) roll requested 2024-11-13T14:29:31,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171967 2024-11-13T14:29:31,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171967 2024-11-13T14:29:31,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171967) roll requested 2024-11-13T14:29:31,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171976 2024-11-13T14:29:31,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171976 2024-11-13T14:29:31,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171976) roll requested 2024-11-13T14:29:31,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171985 2024-11-13T14:29:31,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:31,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171985 2024-11-13T14:29:31,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:31,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:31,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171985) roll requested 2024-11-13T14:29:31,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508171994 2024-11-13T14:29:32,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171994 2024-11-13T14:29:32,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508171994) roll requested 2024-11-13T14:29:32,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172004 2024-11-13T14:29:32,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508171994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172004 2024-11-13T14:29:32,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172004) roll requested 2024-11-13T14:29:32,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172011 2024-11-13T14:29:32,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172011 2024-11-13T14:29:32,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172011) roll requested 2024-11-13T14:29:32,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172017 2024-11-13T14:29:32,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172017 2024-11-13T14:29:32,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172017) roll requested 2024-11-13T14:29:32,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172025 2024-11-13T14:29:32,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172025 2024-11-13T14:29:32,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172025) roll requested 2024-11-13T14:29:32,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172032 2024-11-13T14:29:32,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172032 2024-11-13T14:29:32,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172032) roll requested 2024-11-13T14:29:32,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172038 2024-11-13T14:29:32,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172038 2024-11-13T14:29:32,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172038) roll requested 2024-11-13T14:29:32,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172045 2024-11-13T14:29:32,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172045 2024-11-13T14:29:32,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172045) roll requested 2024-11-13T14:29:32,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172054 2024-11-13T14:29:32,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172054 2024-11-13T14:29:32,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172054) roll requested 2024-11-13T14:29:32,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172062 2024-11-13T14:29:32,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172062 2024-11-13T14:29:32,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172062) roll requested 2024-11-13T14:29:32,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172074 2024-11-13T14:29:32,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172074 2024-11-13T14:29:32,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172074) roll requested 2024-11-13T14:29:32,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172081 2024-11-13T14:29:32,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172081 2024-11-13T14:29:32,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172081) roll requested 2024-11-13T14:29:32,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172086 2024-11-13T14:29:32,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172086 2024-11-13T14:29:32,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172086) roll requested 2024-11-13T14:29:32,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172091 2024-11-13T14:29:32,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172091 2024-11-13T14:29:32,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172091) roll requested 2024-11-13T14:29:32,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172099 2024-11-13T14:29:32,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172099 2024-11-13T14:29:32,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172099) roll requested 2024-11-13T14:29:32,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172107 2024-11-13T14:29:32,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172107 2024-11-13T14:29:32,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172107) roll requested 2024-11-13T14:29:32,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172114 2024-11-13T14:29:32,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172114 2024-11-13T14:29:32,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172114) roll requested 2024-11-13T14:29:32,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172121 2024-11-13T14:29:32,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172121 2024-11-13T14:29:32,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172121) roll requested 2024-11-13T14:29:32,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172129 2024-11-13T14:29:32,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172129 2024-11-13T14:29:32,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172129) roll requested 2024-11-13T14:29:32,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172137 2024-11-13T14:29:32,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172137 2024-11-13T14:29:32,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172137) roll requested 2024-11-13T14:29:32,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172146 2024-11-13T14:29:32,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172146 2024-11-13T14:29:32,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172146) roll requested 2024-11-13T14:29:32,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172153 2024-11-13T14:29:32,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172153 2024-11-13T14:29:32,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172153) roll requested 2024-11-13T14:29:32,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172159 2024-11-13T14:29:32,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172159 2024-11-13T14:29:32,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172159) roll requested 2024-11-13T14:29:32,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172164 2024-11-13T14:29:32,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172164 2024-11-13T14:29:32,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172164) roll requested 2024-11-13T14:29:32,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172171 2024-11-13T14:29:32,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172171 2024-11-13T14:29:32,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172171) roll requested 2024-11-13T14:29:32,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172179 2024-11-13T14:29:32,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172179 2024-11-13T14:29:32,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172179) roll requested 2024-11-13T14:29:32,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172186 2024-11-13T14:29:32,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172186 2024-11-13T14:29:32,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172186) roll requested 2024-11-13T14:29:32,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172194 2024-11-13T14:29:32,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172194 2024-11-13T14:29:32,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172194) roll requested 2024-11-13T14:29:32,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172204 2024-11-13T14:29:32,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172204 2024-11-13T14:29:32,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172204) roll requested 2024-11-13T14:29:32,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172214 2024-11-13T14:29:32,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172214 2024-11-13T14:29:32,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172214) roll requested 2024-11-13T14:29:32,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172220 2024-11-13T14:29:32,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172220 2024-11-13T14:29:32,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172220) roll requested 2024-11-13T14:29:32,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172227 2024-11-13T14:29:32,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172227 2024-11-13T14:29:32,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172227) roll requested 2024-11-13T14:29:32,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172237 2024-11-13T14:29:32,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172237 2024-11-13T14:29:32,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172237) roll requested 2024-11-13T14:29:32,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172245 2024-11-13T14:29:32,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172245 2024-11-13T14:29:32,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172245) roll requested 2024-11-13T14:29:32,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172250 2024-11-13T14:29:32,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172250 2024-11-13T14:29:32,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172250) roll requested 2024-11-13T14:29:32,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172256 2024-11-13T14:29:32,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172256 2024-11-13T14:29:32,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172256) roll requested 2024-11-13T14:29:32,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172264 2024-11-13T14:29:32,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172264 2024-11-13T14:29:32,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172264) roll requested 2024-11-13T14:29:32,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172271 2024-11-13T14:29:32,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172271 2024-11-13T14:29:32,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172271) roll requested 2024-11-13T14:29:32,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172278 2024-11-13T14:29:32,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172278 2024-11-13T14:29:32,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172278) roll requested 2024-11-13T14:29:32,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172285 2024-11-13T14:29:32,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172285 2024-11-13T14:29:32,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172285) roll requested 2024-11-13T14:29:32,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172292 2024-11-13T14:29:32,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172292 2024-11-13T14:29:32,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172292) roll requested 2024-11-13T14:29:32,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172299 2024-11-13T14:29:32,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172299 2024-11-13T14:29:32,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172299) roll requested 2024-11-13T14:29:32,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172307 2024-11-13T14:29:32,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172307 2024-11-13T14:29:32,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172307) roll requested 2024-11-13T14:29:32,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172313 2024-11-13T14:29:32,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172313 2024-11-13T14:29:32,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172313) roll requested 2024-11-13T14:29:32,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172321 2024-11-13T14:29:32,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172321 2024-11-13T14:29:32,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172321) roll requested 2024-11-13T14:29:32,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172330 2024-11-13T14:29:32,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172330 2024-11-13T14:29:32,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172330) roll requested 2024-11-13T14:29:32,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172339 2024-11-13T14:29:32,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172339 2024-11-13T14:29:32,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172339) roll requested 2024-11-13T14:29:32,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172348 2024-11-13T14:29:32,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172348 2024-11-13T14:29:32,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172348) roll requested 2024-11-13T14:29:32,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172361 2024-11-13T14:29:32,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172361 2024-11-13T14:29:32,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172361) roll requested 2024-11-13T14:29:32,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172371 2024-11-13T14:29:32,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172371 2024-11-13T14:29:32,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172371) roll requested 2024-11-13T14:29:32,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172380 2024-11-13T14:29:32,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172380 2024-11-13T14:29:32,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172380) roll requested 2024-11-13T14:29:32,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172388 2024-11-13T14:29:32,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172388 2024-11-13T14:29:32,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172388) roll requested 2024-11-13T14:29:32,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172394 2024-11-13T14:29:32,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172394 2024-11-13T14:29:32,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172394) roll requested 2024-11-13T14:29:32,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172399 2024-11-13T14:29:32,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172399 2024-11-13T14:29:32,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172399) roll requested 2024-11-13T14:29:32,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172406 2024-11-13T14:29:32,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172406 2024-11-13T14:29:32,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172406) roll requested 2024-11-13T14:29:32,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172413 2024-11-13T14:29:32,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172413 2024-11-13T14:29:32,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172413) roll requested 2024-11-13T14:29:32,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172421 2024-11-13T14:29:32,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172421 2024-11-13T14:29:32,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172421) roll requested 2024-11-13T14:29:32,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172429 2024-11-13T14:29:32,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172429 2024-11-13T14:29:32,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172429) roll requested 2024-11-13T14:29:32,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172438 2024-11-13T14:29:32,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172438 2024-11-13T14:29:32,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172438) roll requested 2024-11-13T14:29:32,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172444 2024-11-13T14:29:32,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172444 2024-11-13T14:29:32,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172444) roll requested 2024-11-13T14:29:32,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172450 2024-11-13T14:29:32,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172450 2024-11-13T14:29:32,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172450) roll requested 2024-11-13T14:29:32,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172455 2024-11-13T14:29:32,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172455 2024-11-13T14:29:32,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172455) roll requested 2024-11-13T14:29:32,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172460 2024-11-13T14:29:32,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172460 2024-11-13T14:29:32,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172460) roll requested 2024-11-13T14:29:32,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172465 2024-11-13T14:29:32,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172465 2024-11-13T14:29:32,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172465) roll requested 2024-11-13T14:29:32,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172471 2024-11-13T14:29:32,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172471 2024-11-13T14:29:32,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172471) roll requested 2024-11-13T14:29:32,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172478 2024-11-13T14:29:32,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172478 2024-11-13T14:29:32,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172478) roll requested 2024-11-13T14:29:32,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172485 2024-11-13T14:29:32,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172485 2024-11-13T14:29:32,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172485) roll requested 2024-11-13T14:29:32,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172493 2024-11-13T14:29:32,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172493 2024-11-13T14:29:32,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172493) roll requested 2024-11-13T14:29:32,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172502 2024-11-13T14:29:32,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172502 2024-11-13T14:29:32,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172502) roll requested 2024-11-13T14:29:32,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172512 2024-11-13T14:29:32,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172512 2024-11-13T14:29:32,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172512) roll requested 2024-11-13T14:29:32,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172520 2024-11-13T14:29:32,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172520 2024-11-13T14:29:32,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172520) roll requested 2024-11-13T14:29:32,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172526 2024-11-13T14:29:32,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172526 2024-11-13T14:29:32,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172526) roll requested 2024-11-13T14:29:32,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172532 2024-11-13T14:29:32,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172532 2024-11-13T14:29:32,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172532) roll requested 2024-11-13T14:29:32,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172537 2024-11-13T14:29:32,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172537 2024-11-13T14:29:32,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172537) roll requested 2024-11-13T14:29:32,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172542 2024-11-13T14:29:32,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172542 2024-11-13T14:29:32,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172542) roll requested 2024-11-13T14:29:32,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172547 2024-11-13T14:29:32,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172547 2024-11-13T14:29:32,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172547) roll requested 2024-11-13T14:29:32,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172554 2024-11-13T14:29:32,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172554 2024-11-13T14:29:32,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172554) roll requested 2024-11-13T14:29:32,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172560 2024-11-13T14:29:32,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172560 2024-11-13T14:29:32,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172560) roll requested 2024-11-13T14:29:32,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172567 2024-11-13T14:29:32,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172567 2024-11-13T14:29:32,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172567) roll requested 2024-11-13T14:29:32,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172575 2024-11-13T14:29:32,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172575 2024-11-13T14:29:32,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172575) roll requested 2024-11-13T14:29:32,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172582 2024-11-13T14:29:32,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172582 2024-11-13T14:29:32,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172582) roll requested 2024-11-13T14:29:32,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172589 2024-11-13T14:29:32,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172589 2024-11-13T14:29:32,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172589) roll requested 2024-11-13T14:29:32,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172594 2024-11-13T14:29:32,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172594 2024-11-13T14:29:32,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172594) roll requested 2024-11-13T14:29:32,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172602 2024-11-13T14:29:32,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172602 2024-11-13T14:29:32,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172602) roll requested 2024-11-13T14:29:32,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172611 2024-11-13T14:29:32,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172611 2024-11-13T14:29:32,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172611) roll requested 2024-11-13T14:29:32,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172618 2024-11-13T14:29:32,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172618 2024-11-13T14:29:32,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172618) roll requested 2024-11-13T14:29:32,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172625 2024-11-13T14:29:32,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172625 2024-11-13T14:29:32,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172625) roll requested 2024-11-13T14:29:32,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172632 2024-11-13T14:29:32,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172632 2024-11-13T14:29:32,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172632) roll requested 2024-11-13T14:29:32,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172640 2024-11-13T14:29:32,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172640 2024-11-13T14:29:32,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172640) roll requested 2024-11-13T14:29:32,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172650 2024-11-13T14:29:32,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172650 2024-11-13T14:29:32,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172650) roll requested 2024-11-13T14:29:32,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172657 2024-11-13T14:29:32,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172657 2024-11-13T14:29:32,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172657) roll requested 2024-11-13T14:29:32,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172666 2024-11-13T14:29:32,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172666 2024-11-13T14:29:32,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172666) roll requested 2024-11-13T14:29:32,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172674 2024-11-13T14:29:32,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172674 2024-11-13T14:29:32,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172674) roll requested 2024-11-13T14:29:32,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172682 2024-11-13T14:29:32,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172682 2024-11-13T14:29:32,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172682) roll requested 2024-11-13T14:29:32,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172692 2024-11-13T14:29:32,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172692 2024-11-13T14:29:32,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172692) roll requested 2024-11-13T14:29:32,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172699 2024-11-13T14:29:32,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172699 2024-11-13T14:29:32,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172699) roll requested 2024-11-13T14:29:32,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172706 2024-11-13T14:29:32,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172706 2024-11-13T14:29:32,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172706) roll requested 2024-11-13T14:29:32,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172712 2024-11-13T14:29:32,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172712 2024-11-13T14:29:32,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172712) roll requested 2024-11-13T14:29:32,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172720 2024-11-13T14:29:32,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172720 2024-11-13T14:29:32,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172720) roll requested 2024-11-13T14:29:32,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172729 2024-11-13T14:29:32,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172729 2024-11-13T14:29:32,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172729) roll requested 2024-11-13T14:29:32,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172737 2024-11-13T14:29:32,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172737 2024-11-13T14:29:32,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172737) roll requested 2024-11-13T14:29:32,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172744 2024-11-13T14:29:32,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172744 2024-11-13T14:29:32,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172744) roll requested 2024-11-13T14:29:32,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172751 2024-11-13T14:29:32,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172751 2024-11-13T14:29:32,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172751) roll requested 2024-11-13T14:29:32,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172760 2024-11-13T14:29:32,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172760 2024-11-13T14:29:32,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172760) roll requested 2024-11-13T14:29:32,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172765 2024-11-13T14:29:32,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172765 2024-11-13T14:29:32,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172765) roll requested 2024-11-13T14:29:32,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172771 2024-11-13T14:29:32,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172771 2024-11-13T14:29:32,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172771) roll requested 2024-11-13T14:29:32,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172777 2024-11-13T14:29:32,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172777 2024-11-13T14:29:32,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172777) roll requested 2024-11-13T14:29:32,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172786 2024-11-13T14:29:32,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172786 2024-11-13T14:29:32,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172786) roll requested 2024-11-13T14:29:32,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172798 2024-11-13T14:29:32,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172798 2024-11-13T14:29:32,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172798) roll requested 2024-11-13T14:29:32,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172804 2024-11-13T14:29:32,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172804 2024-11-13T14:29:32,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172804) roll requested 2024-11-13T14:29:32,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172810 2024-11-13T14:29:32,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172810 2024-11-13T14:29:32,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172810) roll requested 2024-11-13T14:29:32,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172816 2024-11-13T14:29:32,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172816 2024-11-13T14:29:32,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172816) roll requested 2024-11-13T14:29:32,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172822 2024-11-13T14:29:32,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172822 2024-11-13T14:29:32,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172822) roll requested 2024-11-13T14:29:32,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172827 2024-11-13T14:29:32,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172827 2024-11-13T14:29:32,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172827) roll requested 2024-11-13T14:29:32,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172832 2024-11-13T14:29:32,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172832 2024-11-13T14:29:32,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172832) roll requested 2024-11-13T14:29:32,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172837 2024-11-13T14:29:32,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172837 2024-11-13T14:29:32,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172837) roll requested 2024-11-13T14:29:32,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172842 2024-11-13T14:29:32,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172842 2024-11-13T14:29:32,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172842) roll requested 2024-11-13T14:29:32,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172848 2024-11-13T14:29:32,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172848 2024-11-13T14:29:32,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172848) roll requested 2024-11-13T14:29:32,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172854 2024-11-13T14:29:32,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172854 2024-11-13T14:29:32,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172854) roll requested 2024-11-13T14:29:32,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172860 2024-11-13T14:29:32,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172860 2024-11-13T14:29:32,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172860) roll requested 2024-11-13T14:29:32,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172866 2024-11-13T14:29:32,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172866 2024-11-13T14:29:32,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172866) roll requested 2024-11-13T14:29:32,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172877 2024-11-13T14:29:32,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172877 2024-11-13T14:29:32,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172877) roll requested 2024-11-13T14:29:32,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172885 2024-11-13T14:29:32,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172885 2024-11-13T14:29:32,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172885) roll requested 2024-11-13T14:29:32,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172894 2024-11-13T14:29:32,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172894 2024-11-13T14:29:32,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172894) roll requested 2024-11-13T14:29:32,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172900 2024-11-13T14:29:32,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172900 2024-11-13T14:29:32,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172900) roll requested 2024-11-13T14:29:32,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172905 2024-11-13T14:29:32,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172905 2024-11-13T14:29:32,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172905) roll requested 2024-11-13T14:29:32,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172911 2024-11-13T14:29:32,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172911 2024-11-13T14:29:32,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172911) roll requested 2024-11-13T14:29:32,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172917 2024-11-13T14:29:32,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172917 2024-11-13T14:29:32,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172917) roll requested 2024-11-13T14:29:32,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172924 2024-11-13T14:29:32,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172924 2024-11-13T14:29:32,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172924) roll requested 2024-11-13T14:29:32,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172932 2024-11-13T14:29:32,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172932 2024-11-13T14:29:32,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172932) roll requested 2024-11-13T14:29:32,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172941 2024-11-13T14:29:32,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172941 2024-11-13T14:29:32,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172941) roll requested 2024-11-13T14:29:32,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172952 2024-11-13T14:29:32,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172952 2024-11-13T14:29:32,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172952) roll requested 2024-11-13T14:29:32,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172960 2024-11-13T14:29:32,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172960 2024-11-13T14:29:32,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172960) roll requested 2024-11-13T14:29:32,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172968 2024-11-13T14:29:32,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172968 2024-11-13T14:29:32,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172968) roll requested 2024-11-13T14:29:32,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172976 2024-11-13T14:29:32,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172976 2024-11-13T14:29:32,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172976) roll requested 2024-11-13T14:29:32,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172985 2024-11-13T14:29:32,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:32,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172985 2024-11-13T14:29:32,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:32,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:32,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172985) roll requested 2024-11-13T14:29:32,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508172995 2024-11-13T14:29:33,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172995 2024-11-13T14:29:33,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508172995) roll requested 2024-11-13T14:29:33,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173003 2024-11-13T14:29:33,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508172995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173003 2024-11-13T14:29:33,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173003) roll requested 2024-11-13T14:29:33,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173008 2024-11-13T14:29:33,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173008 2024-11-13T14:29:33,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173008) roll requested 2024-11-13T14:29:33,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173015 2024-11-13T14:29:33,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173015 2024-11-13T14:29:33,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173015) roll requested 2024-11-13T14:29:33,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173022 2024-11-13T14:29:33,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173022 2024-11-13T14:29:33,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173022) roll requested 2024-11-13T14:29:33,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173030 2024-11-13T14:29:33,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173030 2024-11-13T14:29:33,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173030) roll requested 2024-11-13T14:29:33,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173039 2024-11-13T14:29:33,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173039 2024-11-13T14:29:33,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173039) roll requested 2024-11-13T14:29:33,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173047 2024-11-13T14:29:33,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173047 2024-11-13T14:29:33,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173047) roll requested 2024-11-13T14:29:33,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173055 2024-11-13T14:29:33,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173055 2024-11-13T14:29:33,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173055) roll requested 2024-11-13T14:29:33,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173062 2024-11-13T14:29:33,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173062 2024-11-13T14:29:33,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173062) roll requested 2024-11-13T14:29:33,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173070 2024-11-13T14:29:33,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173070 2024-11-13T14:29:33,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173070) roll requested 2024-11-13T14:29:33,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173079 2024-11-13T14:29:33,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173079 2024-11-13T14:29:33,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173079) roll requested 2024-11-13T14:29:33,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173088 2024-11-13T14:29:33,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173088 2024-11-13T14:29:33,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173088) roll requested 2024-11-13T14:29:33,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173095 2024-11-13T14:29:33,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173095 2024-11-13T14:29:33,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173095) roll requested 2024-11-13T14:29:33,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173103 2024-11-13T14:29:33,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173103 2024-11-13T14:29:33,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173103) roll requested 2024-11-13T14:29:33,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173113 2024-11-13T14:29:33,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173113 2024-11-13T14:29:33,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173113) roll requested 2024-11-13T14:29:33,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173123 2024-11-13T14:29:33,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173123 2024-11-13T14:29:33,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173123) roll requested 2024-11-13T14:29:33,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173134 2024-11-13T14:29:33,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173134 2024-11-13T14:29:33,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173134) roll requested 2024-11-13T14:29:33,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173144 2024-11-13T14:29:33,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173144 2024-11-13T14:29:33,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173144) roll requested 2024-11-13T14:29:33,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173152 2024-11-13T14:29:33,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173152 2024-11-13T14:29:33,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173152) roll requested 2024-11-13T14:29:33,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173160 2024-11-13T14:29:33,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173160 2024-11-13T14:29:33,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173160) roll requested 2024-11-13T14:29:33,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173171 2024-11-13T14:29:33,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173171 2024-11-13T14:29:33,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173171) roll requested 2024-11-13T14:29:33,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173180 2024-11-13T14:29:33,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173180 2024-11-13T14:29:33,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173180) roll requested 2024-11-13T14:29:33,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173188 2024-11-13T14:29:33,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173188 2024-11-13T14:29:33,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173188) roll requested 2024-11-13T14:29:33,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173196 2024-11-13T14:29:33,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173196 2024-11-13T14:29:33,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173196) roll requested 2024-11-13T14:29:33,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173205 2024-11-13T14:29:33,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173205 2024-11-13T14:29:33,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173205) roll requested 2024-11-13T14:29:33,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173212 2024-11-13T14:29:33,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173212 2024-11-13T14:29:33,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173212) roll requested 2024-11-13T14:29:33,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173218 2024-11-13T14:29:33,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173218 2024-11-13T14:29:33,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173218) roll requested 2024-11-13T14:29:33,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173223 2024-11-13T14:29:33,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173223 2024-11-13T14:29:33,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173223) roll requested 2024-11-13T14:29:33,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173227 2024-11-13T14:29:33,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173227 2024-11-13T14:29:33,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173227) roll requested 2024-11-13T14:29:33,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173234 2024-11-13T14:29:33,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173234 2024-11-13T14:29:33,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173234) roll requested 2024-11-13T14:29:33,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173241 2024-11-13T14:29:33,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173241 2024-11-13T14:29:33,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173241) roll requested 2024-11-13T14:29:33,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173248 2024-11-13T14:29:33,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173248 2024-11-13T14:29:33,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173248) roll requested 2024-11-13T14:29:33,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173257 2024-11-13T14:29:33,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173257 2024-11-13T14:29:33,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173257) roll requested 2024-11-13T14:29:33,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173266 2024-11-13T14:29:33,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173266 2024-11-13T14:29:33,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173266) roll requested 2024-11-13T14:29:33,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173276 2024-11-13T14:29:33,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173276 2024-11-13T14:29:33,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173276) roll requested 2024-11-13T14:29:33,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173284 2024-11-13T14:29:33,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173284 2024-11-13T14:29:33,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173284) roll requested 2024-11-13T14:29:33,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173292 2024-11-13T14:29:33,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173292 2024-11-13T14:29:33,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173292) roll requested 2024-11-13T14:29:33,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173300 2024-11-13T14:29:33,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173300 2024-11-13T14:29:33,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173300) roll requested 2024-11-13T14:29:33,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173310 2024-11-13T14:29:33,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173310 2024-11-13T14:29:33,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173310) roll requested 2024-11-13T14:29:33,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173321 2024-11-13T14:29:33,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173321 2024-11-13T14:29:33,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173321) roll requested 2024-11-13T14:29:33,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173331 2024-11-13T14:29:33,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173331 2024-11-13T14:29:33,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173331) roll requested 2024-11-13T14:29:33,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173340 2024-11-13T14:29:33,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173340 2024-11-13T14:29:33,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173340) roll requested 2024-11-13T14:29:33,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173349 2024-11-13T14:29:33,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173349 2024-11-13T14:29:33,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173349) roll requested 2024-11-13T14:29:33,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173359 2024-11-13T14:29:33,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173359 2024-11-13T14:29:33,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173359) roll requested 2024-11-13T14:29:33,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173370 2024-11-13T14:29:33,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173370 2024-11-13T14:29:33,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173370) roll requested 2024-11-13T14:29:33,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173380 2024-11-13T14:29:33,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173380 2024-11-13T14:29:33,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173380) roll requested 2024-11-13T14:29:33,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173389 2024-11-13T14:29:33,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173389 2024-11-13T14:29:33,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173389) roll requested 2024-11-13T14:29:33,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173396 2024-11-13T14:29:33,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173396 2024-11-13T14:29:33,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173396) roll requested 2024-11-13T14:29:33,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173402 2024-11-13T14:29:33,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173402 2024-11-13T14:29:33,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173402) roll requested 2024-11-13T14:29:33,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173410 2024-11-13T14:29:33,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173410 2024-11-13T14:29:33,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173410) roll requested 2024-11-13T14:29:33,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173420 2024-11-13T14:29:33,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173420 2024-11-13T14:29:33,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173420) roll requested 2024-11-13T14:29:33,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173427 2024-11-13T14:29:33,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173427 2024-11-13T14:29:33,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173427) roll requested 2024-11-13T14:29:33,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173436 2024-11-13T14:29:33,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173436 2024-11-13T14:29:33,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173436) roll requested 2024-11-13T14:29:33,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173449 2024-11-13T14:29:33,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173449 2024-11-13T14:29:33,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173449) roll requested 2024-11-13T14:29:33,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173454 2024-11-13T14:29:33,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173454 2024-11-13T14:29:33,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173454) roll requested 2024-11-13T14:29:33,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173459 2024-11-13T14:29:33,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173459 2024-11-13T14:29:33,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173459) roll requested 2024-11-13T14:29:33,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173463 2024-11-13T14:29:33,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173463 2024-11-13T14:29:33,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173463) roll requested 2024-11-13T14:29:33,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173470 2024-11-13T14:29:33,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173470 2024-11-13T14:29:33,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173470) roll requested 2024-11-13T14:29:33,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173477 2024-11-13T14:29:33,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173477 2024-11-13T14:29:33,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173477) roll requested 2024-11-13T14:29:33,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173484 2024-11-13T14:29:33,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173484 2024-11-13T14:29:33,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173484) roll requested 2024-11-13T14:29:33,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173491 2024-11-13T14:29:33,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173491 2024-11-13T14:29:33,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173491) roll requested 2024-11-13T14:29:33,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173500 2024-11-13T14:29:33,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173500 2024-11-13T14:29:33,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173500) roll requested 2024-11-13T14:29:33,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173507 2024-11-13T14:29:33,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173507 2024-11-13T14:29:33,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173507) roll requested 2024-11-13T14:29:33,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173515 2024-11-13T14:29:33,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173515 2024-11-13T14:29:33,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173515) roll requested 2024-11-13T14:29:33,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173524 2024-11-13T14:29:33,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173524 2024-11-13T14:29:33,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173524) roll requested 2024-11-13T14:29:33,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173534 2024-11-13T14:29:33,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173534 2024-11-13T14:29:33,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173534) roll requested 2024-11-13T14:29:33,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173539 2024-11-13T14:29:33,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173539 2024-11-13T14:29:33,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173539) roll requested 2024-11-13T14:29:33,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173544 2024-11-13T14:29:33,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173544 2024-11-13T14:29:33,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173544) roll requested 2024-11-13T14:29:33,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173550 2024-11-13T14:29:33,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173550 2024-11-13T14:29:33,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173550) roll requested 2024-11-13T14:29:33,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173557 2024-11-13T14:29:33,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173557 2024-11-13T14:29:33,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173557) roll requested 2024-11-13T14:29:33,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173566 2024-11-13T14:29:33,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173566 2024-11-13T14:29:33,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173566) roll requested 2024-11-13T14:29:33,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173572 2024-11-13T14:29:33,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173572 2024-11-13T14:29:33,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173572) roll requested 2024-11-13T14:29:33,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173580 2024-11-13T14:29:33,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173580 2024-11-13T14:29:33,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173580) roll requested 2024-11-13T14:29:33,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173588 2024-11-13T14:29:33,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173588 2024-11-13T14:29:33,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173588) roll requested 2024-11-13T14:29:33,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173596 2024-11-13T14:29:33,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173596 2024-11-13T14:29:33,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173596) roll requested 2024-11-13T14:29:33,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173606 2024-11-13T14:29:33,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173606 2024-11-13T14:29:33,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173606) roll requested 2024-11-13T14:29:33,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173616 2024-11-13T14:29:33,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173616 2024-11-13T14:29:33,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173616) roll requested 2024-11-13T14:29:33,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173625 2024-11-13T14:29:33,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173625 2024-11-13T14:29:33,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173625) roll requested 2024-11-13T14:29:33,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173631 2024-11-13T14:29:33,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173631 2024-11-13T14:29:33,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173631) roll requested 2024-11-13T14:29:33,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173638 2024-11-13T14:29:33,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173638 2024-11-13T14:29:33,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173638) roll requested 2024-11-13T14:29:33,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173646 2024-11-13T14:29:33,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173646 2024-11-13T14:29:33,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173646) roll requested 2024-11-13T14:29:33,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173655 2024-11-13T14:29:33,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173655 2024-11-13T14:29:33,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173655) roll requested 2024-11-13T14:29:33,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173663 2024-11-13T14:29:33,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173663 2024-11-13T14:29:33,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173663) roll requested 2024-11-13T14:29:33,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173670 2024-11-13T14:29:33,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173670 2024-11-13T14:29:33,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173670) roll requested 2024-11-13T14:29:33,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173677 2024-11-13T14:29:33,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173677 2024-11-13T14:29:33,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173677) roll requested 2024-11-13T14:29:33,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173684 2024-11-13T14:29:33,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173684 2024-11-13T14:29:33,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173684) roll requested 2024-11-13T14:29:33,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173692 2024-11-13T14:29:33,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173692 2024-11-13T14:29:33,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173692) roll requested 2024-11-13T14:29:33,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173700 2024-11-13T14:29:33,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173700 2024-11-13T14:29:33,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173700) roll requested 2024-11-13T14:29:33,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173709 2024-11-13T14:29:33,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173709 2024-11-13T14:29:33,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173709) roll requested 2024-11-13T14:29:33,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173719 2024-11-13T14:29:33,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173719 2024-11-13T14:29:33,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173719) roll requested 2024-11-13T14:29:33,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173728 2024-11-13T14:29:33,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173728 2024-11-13T14:29:33,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173728) roll requested 2024-11-13T14:29:33,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173738 2024-11-13T14:29:33,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173738 2024-11-13T14:29:33,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173738) roll requested 2024-11-13T14:29:33,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173745 2024-11-13T14:29:33,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173745 2024-11-13T14:29:33,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173745) roll requested 2024-11-13T14:29:33,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173754 2024-11-13T14:29:33,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173754 2024-11-13T14:29:33,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173754) roll requested 2024-11-13T14:29:33,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173764 2024-11-13T14:29:33,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173764 2024-11-13T14:29:33,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173764) roll requested 2024-11-13T14:29:33,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173775 2024-11-13T14:29:33,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173775 2024-11-13T14:29:33,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173775) roll requested 2024-11-13T14:29:33,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173783 2024-11-13T14:29:33,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173783 2024-11-13T14:29:33,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173783) roll requested 2024-11-13T14:29:33,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173789 2024-11-13T14:29:33,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173789 2024-11-13T14:29:33,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173789) roll requested 2024-11-13T14:29:33,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173797 2024-11-13T14:29:33,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173797 2024-11-13T14:29:33,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173797) roll requested 2024-11-13T14:29:33,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173805 2024-11-13T14:29:33,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173805 2024-11-13T14:29:33,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173805) roll requested 2024-11-13T14:29:33,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173815 2024-11-13T14:29:33,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173815 2024-11-13T14:29:33,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173815) roll requested 2024-11-13T14:29:33,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173824 2024-11-13T14:29:33,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173824 2024-11-13T14:29:33,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173824) roll requested 2024-11-13T14:29:33,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173832 2024-11-13T14:29:33,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173832 2024-11-13T14:29:33,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173832) roll requested 2024-11-13T14:29:33,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173840 2024-11-13T14:29:33,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173840 2024-11-13T14:29:33,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173840) roll requested 2024-11-13T14:29:33,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173847 2024-11-13T14:29:33,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173847 2024-11-13T14:29:33,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173847) roll requested 2024-11-13T14:29:33,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173853 2024-11-13T14:29:33,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173853 2024-11-13T14:29:33,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173853) roll requested 2024-11-13T14:29:33,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173860 2024-11-13T14:29:33,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173860 2024-11-13T14:29:33,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173860) roll requested 2024-11-13T14:29:33,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173867 2024-11-13T14:29:33,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173867 2024-11-13T14:29:33,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173867) roll requested 2024-11-13T14:29:33,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173877 2024-11-13T14:29:33,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173877 2024-11-13T14:29:33,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173877) roll requested 2024-11-13T14:29:33,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173887 2024-11-13T14:29:33,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173887 2024-11-13T14:29:33,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173887) roll requested 2024-11-13T14:29:33,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173896 2024-11-13T14:29:33,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173896 2024-11-13T14:29:33,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173896) roll requested 2024-11-13T14:29:33,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173902 2024-11-13T14:29:33,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173902 2024-11-13T14:29:33,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173902) roll requested 2024-11-13T14:29:33,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173909 2024-11-13T14:29:33,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173909 2024-11-13T14:29:33,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173909) roll requested 2024-11-13T14:29:33,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173917 2024-11-13T14:29:33,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173917 2024-11-13T14:29:33,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173917) roll requested 2024-11-13T14:29:33,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173926 2024-11-13T14:29:33,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173926 2024-11-13T14:29:33,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173926) roll requested 2024-11-13T14:29:33,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173934 2024-11-13T14:29:33,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173934 2024-11-13T14:29:33,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173934) roll requested 2024-11-13T14:29:33,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173942 2024-11-13T14:29:33,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173942 2024-11-13T14:29:33,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173942) roll requested 2024-11-13T14:29:33,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173952 2024-11-13T14:29:33,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173952 2024-11-13T14:29:33,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173952) roll requested 2024-11-13T14:29:33,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173961 2024-11-13T14:29:33,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173961 2024-11-13T14:29:33,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173961) roll requested 2024-11-13T14:29:33,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173970 2024-11-13T14:29:33,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173970 2024-11-13T14:29:33,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173970) roll requested 2024-11-13T14:29:33,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173979 2024-11-13T14:29:33,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173979 2024-11-13T14:29:33,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173979) roll requested 2024-11-13T14:29:33,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173988 2024-11-13T14:29:33,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:33,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173988 2024-11-13T14:29:33,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:33,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:33,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173988) roll requested 2024-11-13T14:29:33,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508173995 2024-11-13T14:29:34,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173995 2024-11-13T14:29:34,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508173995) roll requested 2024-11-13T14:29:34,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174003 2024-11-13T14:29:34,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508173995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174003 2024-11-13T14:29:34,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174003) roll requested 2024-11-13T14:29:34,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174011 2024-11-13T14:29:34,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174011 2024-11-13T14:29:34,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174011) roll requested 2024-11-13T14:29:34,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174019 2024-11-13T14:29:34,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174019 2024-11-13T14:29:34,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174019) roll requested 2024-11-13T14:29:34,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174026 2024-11-13T14:29:34,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174026 2024-11-13T14:29:34,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174026) roll requested 2024-11-13T14:29:34,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174033 2024-11-13T14:29:34,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174033 2024-11-13T14:29:34,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174033) roll requested 2024-11-13T14:29:34,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174040 2024-11-13T14:29:34,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174040 2024-11-13T14:29:34,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174040) roll requested 2024-11-13T14:29:34,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174048 2024-11-13T14:29:34,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174048 2024-11-13T14:29:34,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174048) roll requested 2024-11-13T14:29:34,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174055 2024-11-13T14:29:34,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174055 2024-11-13T14:29:34,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174055) roll requested 2024-11-13T14:29:34,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174065 2024-11-13T14:29:34,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174065 2024-11-13T14:29:34,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174065) roll requested 2024-11-13T14:29:34,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174074 2024-11-13T14:29:34,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174074 2024-11-13T14:29:34,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174074) roll requested 2024-11-13T14:29:34,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174082 2024-11-13T14:29:34,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174082 2024-11-13T14:29:34,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174082) roll requested 2024-11-13T14:29:34,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174090 2024-11-13T14:29:34,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174090 2024-11-13T14:29:34,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174090) roll requested 2024-11-13T14:29:34,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174098 2024-11-13T14:29:34,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174098 2024-11-13T14:29:34,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174098) roll requested 2024-11-13T14:29:34,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174106 2024-11-13T14:29:34,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174106 2024-11-13T14:29:34,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174106) roll requested 2024-11-13T14:29:34,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174115 2024-11-13T14:29:34,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174115 2024-11-13T14:29:34,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174115) roll requested 2024-11-13T14:29:34,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174124 2024-11-13T14:29:34,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174124 2024-11-13T14:29:34,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174124) roll requested 2024-11-13T14:29:34,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174133 2024-11-13T14:29:34,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174133 2024-11-13T14:29:34,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174133) roll requested 2024-11-13T14:29:34,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174140 2024-11-13T14:29:34,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174140 2024-11-13T14:29:34,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174140) roll requested 2024-11-13T14:29:34,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174148 2024-11-13T14:29:34,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174148 2024-11-13T14:29:34,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174148) roll requested 2024-11-13T14:29:34,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174157 2024-11-13T14:29:34,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174157 2024-11-13T14:29:34,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174157) roll requested 2024-11-13T14:29:34,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174163 2024-11-13T14:29:34,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174163 2024-11-13T14:29:34,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174163) roll requested 2024-11-13T14:29:34,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174169 2024-11-13T14:29:34,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174169 2024-11-13T14:29:34,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174169) roll requested 2024-11-13T14:29:34,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174176 2024-11-13T14:29:34,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174176 2024-11-13T14:29:34,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174176) roll requested 2024-11-13T14:29:34,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174184 2024-11-13T14:29:34,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174184 2024-11-13T14:29:34,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174184) roll requested 2024-11-13T14:29:34,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174191 2024-11-13T14:29:34,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174191 2024-11-13T14:29:34,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174191) roll requested 2024-11-13T14:29:34,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174199 2024-11-13T14:29:34,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174199 2024-11-13T14:29:34,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174199) roll requested 2024-11-13T14:29:34,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174207 2024-11-13T14:29:34,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174207 2024-11-13T14:29:34,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174207) roll requested 2024-11-13T14:29:34,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174214 2024-11-13T14:29:34,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174214 2024-11-13T14:29:34,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174214) roll requested 2024-11-13T14:29:34,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174224 2024-11-13T14:29:34,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174224 2024-11-13T14:29:34,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174224) roll requested 2024-11-13T14:29:34,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174231 2024-11-13T14:29:34,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174231 2024-11-13T14:29:34,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174231) roll requested 2024-11-13T14:29:34,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174240 2024-11-13T14:29:34,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174240 2024-11-13T14:29:34,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174240) roll requested 2024-11-13T14:29:34,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174248 2024-11-13T14:29:34,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174248 2024-11-13T14:29:34,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174248) roll requested 2024-11-13T14:29:34,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174254 2024-11-13T14:29:34,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174254 2024-11-13T14:29:34,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174254) roll requested 2024-11-13T14:29:34,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174259 2024-11-13T14:29:34,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174259 2024-11-13T14:29:34,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174259) roll requested 2024-11-13T14:29:34,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174265 2024-11-13T14:29:34,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174265 2024-11-13T14:29:34,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174265) roll requested 2024-11-13T14:29:34,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174271 2024-11-13T14:29:34,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174271 2024-11-13T14:29:34,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174271) roll requested 2024-11-13T14:29:34,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174278 2024-11-13T14:29:34,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174278 2024-11-13T14:29:34,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174278) roll requested 2024-11-13T14:29:34,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174284 2024-11-13T14:29:34,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174284 2024-11-13T14:29:34,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174284) roll requested 2024-11-13T14:29:34,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174292 2024-11-13T14:29:34,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174292 2024-11-13T14:29:34,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174292) roll requested 2024-11-13T14:29:34,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174299 2024-11-13T14:29:34,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174299 2024-11-13T14:29:34,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174299) roll requested 2024-11-13T14:29:34,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174308 2024-11-13T14:29:34,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174308 2024-11-13T14:29:34,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174308) roll requested 2024-11-13T14:29:34,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174317 2024-11-13T14:29:34,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174317 2024-11-13T14:29:34,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174317) roll requested 2024-11-13T14:29:34,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174327 2024-11-13T14:29:34,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174327 2024-11-13T14:29:34,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174327) roll requested 2024-11-13T14:29:34,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174337 2024-11-13T14:29:34,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174337 2024-11-13T14:29:34,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174337) roll requested 2024-11-13T14:29:34,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174347 2024-11-13T14:29:34,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174347 2024-11-13T14:29:34,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174347) roll requested 2024-11-13T14:29:34,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174355 2024-11-13T14:29:34,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174355 2024-11-13T14:29:34,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174355) roll requested 2024-11-13T14:29:34,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174362 2024-11-13T14:29:34,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174362 2024-11-13T14:29:34,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174362) roll requested 2024-11-13T14:29:34,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174372 2024-11-13T14:29:34,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174372 2024-11-13T14:29:34,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174372) roll requested 2024-11-13T14:29:34,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174380 2024-11-13T14:29:34,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174380 2024-11-13T14:29:34,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174380) roll requested 2024-11-13T14:29:34,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174387 2024-11-13T14:29:34,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174387 2024-11-13T14:29:34,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174387) roll requested 2024-11-13T14:29:34,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174392 2024-11-13T14:29:34,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174392 2024-11-13T14:29:34,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174392) roll requested 2024-11-13T14:29:34,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174397 2024-11-13T14:29:34,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174397 2024-11-13T14:29:34,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174397) roll requested 2024-11-13T14:29:34,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174402 2024-11-13T14:29:34,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174402 2024-11-13T14:29:34,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174402) roll requested 2024-11-13T14:29:34,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174408 2024-11-13T14:29:34,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174408 2024-11-13T14:29:34,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174408) roll requested 2024-11-13T14:29:34,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174415 2024-11-13T14:29:34,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174415 2024-11-13T14:29:34,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174415) roll requested 2024-11-13T14:29:34,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174423 2024-11-13T14:29:34,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174423 2024-11-13T14:29:34,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174423) roll requested 2024-11-13T14:29:34,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174431 2024-11-13T14:29:34,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174431 2024-11-13T14:29:34,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174431) roll requested 2024-11-13T14:29:34,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174440 2024-11-13T14:29:34,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174440 2024-11-13T14:29:34,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174440) roll requested 2024-11-13T14:29:34,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174450 2024-11-13T14:29:34,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174450 2024-11-13T14:29:34,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174450) roll requested 2024-11-13T14:29:34,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174460 2024-11-13T14:29:34,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174460 2024-11-13T14:29:34,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174460) roll requested 2024-11-13T14:29:34,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174468 2024-11-13T14:29:34,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174468 2024-11-13T14:29:34,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174468) roll requested 2024-11-13T14:29:34,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174476 2024-11-13T14:29:34,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174476 2024-11-13T14:29:34,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174476) roll requested 2024-11-13T14:29:34,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174485 2024-11-13T14:29:34,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174485 2024-11-13T14:29:34,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174485) roll requested 2024-11-13T14:29:34,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174495 2024-11-13T14:29:34,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174495 2024-11-13T14:29:34,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174495) roll requested 2024-11-13T14:29:34,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174503 2024-11-13T14:29:34,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174503 2024-11-13T14:29:34,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174503) roll requested 2024-11-13T14:29:34,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174509 2024-11-13T14:29:34,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174509 2024-11-13T14:29:34,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174509) roll requested 2024-11-13T14:29:34,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174515 2024-11-13T14:29:34,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174515 2024-11-13T14:29:34,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174515) roll requested 2024-11-13T14:29:34,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174522 2024-11-13T14:29:34,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174522 2024-11-13T14:29:34,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174522) roll requested 2024-11-13T14:29:34,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174530 2024-11-13T14:29:34,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174530 2024-11-13T14:29:34,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174530) roll requested 2024-11-13T14:29:34,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174540 2024-11-13T14:29:34,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174540 2024-11-13T14:29:34,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174540) roll requested 2024-11-13T14:29:34,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174544 2024-11-13T14:29:34,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174544 2024-11-13T14:29:34,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174544) roll requested 2024-11-13T14:29:34,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174549 2024-11-13T14:29:34,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174549 2024-11-13T14:29:34,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174549) roll requested 2024-11-13T14:29:34,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174557 2024-11-13T14:29:34,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174557 2024-11-13T14:29:34,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174557) roll requested 2024-11-13T14:29:34,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174565 2024-11-13T14:29:34,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174565 2024-11-13T14:29:34,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174565) roll requested 2024-11-13T14:29:34,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174572 2024-11-13T14:29:34,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174572 2024-11-13T14:29:34,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174572) roll requested 2024-11-13T14:29:34,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174580 2024-11-13T14:29:34,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174580 2024-11-13T14:29:34,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174580) roll requested 2024-11-13T14:29:34,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174588 2024-11-13T14:29:34,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174588 2024-11-13T14:29:34,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174588) roll requested 2024-11-13T14:29:34,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174595 2024-11-13T14:29:34,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174595 2024-11-13T14:29:34,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174595) roll requested 2024-11-13T14:29:34,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174603 2024-11-13T14:29:34,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174603 2024-11-13T14:29:34,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174603) roll requested 2024-11-13T14:29:34,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174611 2024-11-13T14:29:34,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174611 2024-11-13T14:29:34,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174611) roll requested 2024-11-13T14:29:34,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174620 2024-11-13T14:29:34,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174620 2024-11-13T14:29:34,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174620) roll requested 2024-11-13T14:29:34,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174629 2024-11-13T14:29:34,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174629 2024-11-13T14:29:34,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174629) roll requested 2024-11-13T14:29:34,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174637 2024-11-13T14:29:34,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174637 2024-11-13T14:29:34,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174637) roll requested 2024-11-13T14:29:34,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174644 2024-11-13T14:29:34,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174644 2024-11-13T14:29:34,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174644) roll requested 2024-11-13T14:29:34,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174653 2024-11-13T14:29:34,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174653 2024-11-13T14:29:34,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174653) roll requested 2024-11-13T14:29:34,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174663 2024-11-13T14:29:34,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174663 2024-11-13T14:29:34,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174663) roll requested 2024-11-13T14:29:34,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174673 2024-11-13T14:29:34,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174673 2024-11-13T14:29:34,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174673) roll requested 2024-11-13T14:29:34,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174683 2024-11-13T14:29:34,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174683 2024-11-13T14:29:34,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174683) roll requested 2024-11-13T14:29:34,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174692 2024-11-13T14:29:34,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174692 2024-11-13T14:29:34,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174692) roll requested 2024-11-13T14:29:34,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174700 2024-11-13T14:29:34,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174700 2024-11-13T14:29:34,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174700) roll requested 2024-11-13T14:29:34,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174708 2024-11-13T14:29:34,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174708 2024-11-13T14:29:34,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174708) roll requested 2024-11-13T14:29:34,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174714 2024-11-13T14:29:34,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174714 2024-11-13T14:29:34,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174714) roll requested 2024-11-13T14:29:34,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174720 2024-11-13T14:29:34,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174720 2024-11-13T14:29:34,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174720) roll requested 2024-11-13T14:29:34,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174725 2024-11-13T14:29:34,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174725 2024-11-13T14:29:34,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174725) roll requested 2024-11-13T14:29:34,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174730 2024-11-13T14:29:34,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174730 2024-11-13T14:29:34,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174730) roll requested 2024-11-13T14:29:34,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174736 2024-11-13T14:29:34,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174736 2024-11-13T14:29:34,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174736) roll requested 2024-11-13T14:29:34,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174745 2024-11-13T14:29:34,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174745 2024-11-13T14:29:34,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174745) roll requested 2024-11-13T14:29:34,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174751 2024-11-13T14:29:34,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174751 2024-11-13T14:29:34,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174751) roll requested 2024-11-13T14:29:34,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174757 2024-11-13T14:29:34,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174757 2024-11-13T14:29:34,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174757) roll requested 2024-11-13T14:29:34,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174764 2024-11-13T14:29:34,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174764 2024-11-13T14:29:34,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174764) roll requested 2024-11-13T14:29:34,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174772 2024-11-13T14:29:34,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174772 2024-11-13T14:29:34,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174772) roll requested 2024-11-13T14:29:34,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174782 2024-11-13T14:29:34,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174782 2024-11-13T14:29:34,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174782) roll requested 2024-11-13T14:29:34,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174791 2024-11-13T14:29:34,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174791 2024-11-13T14:29:34,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174791) roll requested 2024-11-13T14:29:34,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174800 2024-11-13T14:29:34,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174800 2024-11-13T14:29:34,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174800) roll requested 2024-11-13T14:29:34,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174808 2024-11-13T14:29:34,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174808 2024-11-13T14:29:34,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174808) roll requested 2024-11-13T14:29:34,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174814 2024-11-13T14:29:34,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174814 2024-11-13T14:29:34,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174814) roll requested 2024-11-13T14:29:34,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174821 2024-11-13T14:29:34,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174821 2024-11-13T14:29:34,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174821) roll requested 2024-11-13T14:29:34,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174832 2024-11-13T14:29:34,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174832 2024-11-13T14:29:34,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174832) roll requested 2024-11-13T14:29:34,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174840 2024-11-13T14:29:34,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174840 2024-11-13T14:29:34,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174840) roll requested 2024-11-13T14:29:34,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174849 2024-11-13T14:29:34,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174849 2024-11-13T14:29:34,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174849) roll requested 2024-11-13T14:29:34,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174858 2024-11-13T14:29:34,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174858 2024-11-13T14:29:34,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174858) roll requested 2024-11-13T14:29:34,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174868 2024-11-13T14:29:34,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174868 2024-11-13T14:29:34,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174868) roll requested 2024-11-13T14:29:34,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174879 2024-11-13T14:29:34,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174879 2024-11-13T14:29:34,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174879) roll requested 2024-11-13T14:29:34,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174888 2024-11-13T14:29:34,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174888 2024-11-13T14:29:34,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174888) roll requested 2024-11-13T14:29:34,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174896 2024-11-13T14:29:34,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174896 2024-11-13T14:29:34,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174896) roll requested 2024-11-13T14:29:34,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174905 2024-11-13T14:29:34,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174905 2024-11-13T14:29:34,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174905) roll requested 2024-11-13T14:29:34,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174914 2024-11-13T14:29:34,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174914 2024-11-13T14:29:34,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174914) roll requested 2024-11-13T14:29:34,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174923 2024-11-13T14:29:34,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174923 2024-11-13T14:29:34,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174923) roll requested 2024-11-13T14:29:34,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174932 2024-11-13T14:29:34,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174932 2024-11-13T14:29:34,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174932) roll requested 2024-11-13T14:29:34,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174938 2024-11-13T14:29:34,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174938 2024-11-13T14:29:34,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174938) roll requested 2024-11-13T14:29:34,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174944 2024-11-13T14:29:34,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174944 2024-11-13T14:29:34,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174944) roll requested 2024-11-13T14:29:34,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174952 2024-11-13T14:29:34,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174952 2024-11-13T14:29:34,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174952) roll requested 2024-11-13T14:29:34,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174960 2024-11-13T14:29:34,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174960 2024-11-13T14:29:34,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174960) roll requested 2024-11-13T14:29:34,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174969 2024-11-13T14:29:34,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174969 2024-11-13T14:29:34,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174969) roll requested 2024-11-13T14:29:34,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174978 2024-11-13T14:29:34,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174978 2024-11-13T14:29:34,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174978) roll requested 2024-11-13T14:29:34,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174987 2024-11-13T14:29:34,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:34,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174987 2024-11-13T14:29:34,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:34,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:34,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174987) roll requested 2024-11-13T14:29:34,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508174998 2024-11-13T14:29:35,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174998 2024-11-13T14:29:35,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508174998) roll requested 2024-11-13T14:29:35,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175006 2024-11-13T14:29:35,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508174998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175006 2024-11-13T14:29:35,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175006) roll requested 2024-11-13T14:29:35,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175013 2024-11-13T14:29:35,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175013 2024-11-13T14:29:35,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175013) roll requested 2024-11-13T14:29:35,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175022 2024-11-13T14:29:35,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175022 2024-11-13T14:29:35,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175022) roll requested 2024-11-13T14:29:35,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175029 2024-11-13T14:29:35,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175029 2024-11-13T14:29:35,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175029) roll requested 2024-11-13T14:29:35,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175039 2024-11-13T14:29:35,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175039 2024-11-13T14:29:35,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175039) roll requested 2024-11-13T14:29:35,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175048 2024-11-13T14:29:35,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175048 2024-11-13T14:29:35,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175048) roll requested 2024-11-13T14:29:35,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175056 2024-11-13T14:29:35,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175056 2024-11-13T14:29:35,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175056) roll requested 2024-11-13T14:29:35,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175062 2024-11-13T14:29:35,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175062 2024-11-13T14:29:35,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175062) roll requested 2024-11-13T14:29:35,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175070 2024-11-13T14:29:35,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175070 2024-11-13T14:29:35,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175070) roll requested 2024-11-13T14:29:35,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175078 2024-11-13T14:29:35,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175078 2024-11-13T14:29:35,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175078) roll requested 2024-11-13T14:29:35,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175087 2024-11-13T14:29:35,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175087 2024-11-13T14:29:35,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175087) roll requested 2024-11-13T14:29:35,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175096 2024-11-13T14:29:35,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175096 2024-11-13T14:29:35,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175096) roll requested 2024-11-13T14:29:35,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175105 2024-11-13T14:29:35,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175105 2024-11-13T14:29:35,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175105) roll requested 2024-11-13T14:29:35,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175112 2024-11-13T14:29:35,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175112 2024-11-13T14:29:35,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175112) roll requested 2024-11-13T14:29:35,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175120 2024-11-13T14:29:35,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175120 2024-11-13T14:29:35,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175120) roll requested 2024-11-13T14:29:35,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175129 2024-11-13T14:29:35,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175129 2024-11-13T14:29:35,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175129) roll requested 2024-11-13T14:29:35,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175137 2024-11-13T14:29:35,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175137 2024-11-13T14:29:35,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175137) roll requested 2024-11-13T14:29:35,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175145 2024-11-13T14:29:35,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175145 2024-11-13T14:29:35,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175145) roll requested 2024-11-13T14:29:35,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175156 2024-11-13T14:29:35,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175156 2024-11-13T14:29:35,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175156) roll requested 2024-11-13T14:29:35,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175161 2024-11-13T14:29:35,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175161 2024-11-13T14:29:35,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175161) roll requested 2024-11-13T14:29:35,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175166 2024-11-13T14:29:35,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175166 2024-11-13T14:29:35,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175166) roll requested 2024-11-13T14:29:35,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175171 2024-11-13T14:29:35,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175171 2024-11-13T14:29:35,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175171) roll requested 2024-11-13T14:29:35,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175178 2024-11-13T14:29:35,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175178 2024-11-13T14:29:35,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175178) roll requested 2024-11-13T14:29:35,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175185 2024-11-13T14:29:35,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175185 2024-11-13T14:29:35,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175185) roll requested 2024-11-13T14:29:35,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175191 2024-11-13T14:29:35,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175191 2024-11-13T14:29:35,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175191) roll requested 2024-11-13T14:29:35,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175197 2024-11-13T14:29:35,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175197 2024-11-13T14:29:35,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175197) roll requested 2024-11-13T14:29:35,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175204 2024-11-13T14:29:35,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175204 2024-11-13T14:29:35,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175204) roll requested 2024-11-13T14:29:35,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175211 2024-11-13T14:29:35,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175211 2024-11-13T14:29:35,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175211) roll requested 2024-11-13T14:29:35,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175219 2024-11-13T14:29:35,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175219 2024-11-13T14:29:35,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175219) roll requested 2024-11-13T14:29:35,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175229 2024-11-13T14:29:35,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175229 2024-11-13T14:29:35,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175229) roll requested 2024-11-13T14:29:35,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175239 2024-11-13T14:29:35,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175239 2024-11-13T14:29:35,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175239) roll requested 2024-11-13T14:29:35,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175247 2024-11-13T14:29:35,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175247 2024-11-13T14:29:35,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175247) roll requested 2024-11-13T14:29:35,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175252 2024-11-13T14:29:35,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175252 2024-11-13T14:29:35,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175252) roll requested 2024-11-13T14:29:35,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175259 2024-11-13T14:29:35,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175259 2024-11-13T14:29:35,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175259) roll requested 2024-11-13T14:29:35,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175267 2024-11-13T14:29:35,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175267 2024-11-13T14:29:35,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175267) roll requested 2024-11-13T14:29:35,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175276 2024-11-13T14:29:35,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175276 2024-11-13T14:29:35,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175276) roll requested 2024-11-13T14:29:35,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175286 2024-11-13T14:29:35,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175286 2024-11-13T14:29:35,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175286) roll requested 2024-11-13T14:29:35,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175297 2024-11-13T14:29:35,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175297 2024-11-13T14:29:35,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175297) roll requested 2024-11-13T14:29:35,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175306 2024-11-13T14:29:35,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175306 2024-11-13T14:29:35,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175306) roll requested 2024-11-13T14:29:35,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175314 2024-11-13T14:29:35,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175314 2024-11-13T14:29:35,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175314) roll requested 2024-11-13T14:29:35,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175321 2024-11-13T14:29:35,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175321 2024-11-13T14:29:35,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175321) roll requested 2024-11-13T14:29:35,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175328 2024-11-13T14:29:35,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175328 2024-11-13T14:29:35,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175328) roll requested 2024-11-13T14:29:35,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175335 2024-11-13T14:29:35,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175335 2024-11-13T14:29:35,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175335) roll requested 2024-11-13T14:29:35,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175342 2024-11-13T14:29:35,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175342 2024-11-13T14:29:35,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175342) roll requested 2024-11-13T14:29:35,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175350 2024-11-13T14:29:35,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175350 2024-11-13T14:29:35,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175350) roll requested 2024-11-13T14:29:35,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175359 2024-11-13T14:29:35,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175359 2024-11-13T14:29:35,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175359) roll requested 2024-11-13T14:29:35,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175369 2024-11-13T14:29:35,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175369 2024-11-13T14:29:35,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175369) roll requested 2024-11-13T14:29:35,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175378 2024-11-13T14:29:35,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175378 2024-11-13T14:29:35,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175378) roll requested 2024-11-13T14:29:35,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175388 2024-11-13T14:29:35,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175388 2024-11-13T14:29:35,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175388) roll requested 2024-11-13T14:29:35,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175396 2024-11-13T14:29:35,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175396 2024-11-13T14:29:35,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175396) roll requested 2024-11-13T14:29:35,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175405 2024-11-13T14:29:35,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175405 2024-11-13T14:29:35,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175405) roll requested 2024-11-13T14:29:35,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175413 2024-11-13T14:29:35,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175413 2024-11-13T14:29:35,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175413) roll requested 2024-11-13T14:29:35,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175420 2024-11-13T14:29:35,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175420 2024-11-13T14:29:35,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175420) roll requested 2024-11-13T14:29:35,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175426 2024-11-13T14:29:35,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175426 2024-11-13T14:29:35,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175426) roll requested 2024-11-13T14:29:35,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175433 2024-11-13T14:29:35,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175433 2024-11-13T14:29:35,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175433) roll requested 2024-11-13T14:29:35,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175442 2024-11-13T14:29:35,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175442 2024-11-13T14:29:35,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175442) roll requested 2024-11-13T14:29:35,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175450 2024-11-13T14:29:35,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175450 2024-11-13T14:29:35,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175450) roll requested 2024-11-13T14:29:35,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175458 2024-11-13T14:29:35,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175458 2024-11-13T14:29:35,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175458) roll requested 2024-11-13T14:29:35,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175468 2024-11-13T14:29:35,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175468 2024-11-13T14:29:35,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175468) roll requested 2024-11-13T14:29:35,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175475 2024-11-13T14:29:35,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175475 2024-11-13T14:29:35,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175475) roll requested 2024-11-13T14:29:35,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175481 2024-11-13T14:29:35,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175481 2024-11-13T14:29:35,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175481) roll requested 2024-11-13T14:29:35,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175487 2024-11-13T14:29:35,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175487 2024-11-13T14:29:35,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175487) roll requested 2024-11-13T14:29:35,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175495 2024-11-13T14:29:35,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175495 2024-11-13T14:29:35,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175495) roll requested 2024-11-13T14:29:35,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175502 2024-11-13T14:29:35,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175502 2024-11-13T14:29:35,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175502) roll requested 2024-11-13T14:29:35,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175510 2024-11-13T14:29:35,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175510 2024-11-13T14:29:35,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175510) roll requested 2024-11-13T14:29:35,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175519 2024-11-13T14:29:35,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175519 2024-11-13T14:29:35,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175519) roll requested 2024-11-13T14:29:35,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175530 2024-11-13T14:29:35,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175530 2024-11-13T14:29:35,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175530) roll requested 2024-11-13T14:29:35,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175538 2024-11-13T14:29:35,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175538 2024-11-13T14:29:35,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175538) roll requested 2024-11-13T14:29:35,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175543 2024-11-13T14:29:35,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175543 2024-11-13T14:29:35,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175543) roll requested 2024-11-13T14:29:35,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175549 2024-11-13T14:29:35,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175549 2024-11-13T14:29:35,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175549) roll requested 2024-11-13T14:29:35,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175555 2024-11-13T14:29:35,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175555 2024-11-13T14:29:35,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175555) roll requested 2024-11-13T14:29:35,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175563 2024-11-13T14:29:35,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175563 2024-11-13T14:29:35,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175563) roll requested 2024-11-13T14:29:35,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175569 2024-11-13T14:29:35,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175569 2024-11-13T14:29:35,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175569) roll requested 2024-11-13T14:29:35,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175577 2024-11-13T14:29:35,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175577 2024-11-13T14:29:35,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175577) roll requested 2024-11-13T14:29:35,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175586 2024-11-13T14:29:35,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175586 2024-11-13T14:29:35,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175586) roll requested 2024-11-13T14:29:35,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175595 2024-11-13T14:29:35,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175595 2024-11-13T14:29:35,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175595) roll requested 2024-11-13T14:29:35,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175604 2024-11-13T14:29:35,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175604 2024-11-13T14:29:35,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175604) roll requested 2024-11-13T14:29:35,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175615 2024-11-13T14:29:35,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175615 2024-11-13T14:29:35,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175615) roll requested 2024-11-13T14:29:35,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175625 2024-11-13T14:29:35,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175625 2024-11-13T14:29:35,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175625) roll requested 2024-11-13T14:29:35,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175635 2024-11-13T14:29:35,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175635 2024-11-13T14:29:35,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175635) roll requested 2024-11-13T14:29:35,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175648 2024-11-13T14:29:35,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175648 2024-11-13T14:29:35,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175648) roll requested 2024-11-13T14:29:35,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175655 2024-11-13T14:29:35,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175655 2024-11-13T14:29:35,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175655) roll requested 2024-11-13T14:29:35,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175660 2024-11-13T14:29:35,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175660 2024-11-13T14:29:35,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175660) roll requested 2024-11-13T14:29:35,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175667 2024-11-13T14:29:35,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175667 2024-11-13T14:29:35,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175667) roll requested 2024-11-13T14:29:35,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175673 2024-11-13T14:29:35,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175673 2024-11-13T14:29:35,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175673) roll requested 2024-11-13T14:29:35,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175746 2024-11-13T14:29:35,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175746 2024-11-13T14:29:35,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175746) roll requested 2024-11-13T14:29:35,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175753 2024-11-13T14:29:35,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175753 2024-11-13T14:29:35,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175753) roll requested 2024-11-13T14:29:35,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175761 2024-11-13T14:29:35,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175761 2024-11-13T14:29:35,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175761) roll requested 2024-11-13T14:29:35,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175770 2024-11-13T14:29:35,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175770 2024-11-13T14:29:35,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175770) roll requested 2024-11-13T14:29:35,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175778 2024-11-13T14:29:35,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175778 2024-11-13T14:29:35,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175778) roll requested 2024-11-13T14:29:35,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175784 2024-11-13T14:29:35,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175784 2024-11-13T14:29:35,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175784) roll requested 2024-11-13T14:29:35,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175791 2024-11-13T14:29:35,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175791 2024-11-13T14:29:35,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175791) roll requested 2024-11-13T14:29:35,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175800 2024-11-13T14:29:35,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175800 2024-11-13T14:29:35,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175800) roll requested 2024-11-13T14:29:35,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175806 2024-11-13T14:29:35,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175806 2024-11-13T14:29:35,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175806) roll requested 2024-11-13T14:29:35,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175812 2024-11-13T14:29:35,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175812 2024-11-13T14:29:35,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175812) roll requested 2024-11-13T14:29:35,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175817 2024-11-13T14:29:35,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175817 2024-11-13T14:29:35,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175817) roll requested 2024-11-13T14:29:35,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175825 2024-11-13T14:29:35,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175825 2024-11-13T14:29:35,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175825) roll requested 2024-11-13T14:29:35,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175830 2024-11-13T14:29:35,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175830 2024-11-13T14:29:35,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175830) roll requested 2024-11-13T14:29:35,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175835 2024-11-13T14:29:35,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175835 2024-11-13T14:29:35,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175835) roll requested 2024-11-13T14:29:35,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175841 2024-11-13T14:29:35,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175841 2024-11-13T14:29:35,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175841) roll requested 2024-11-13T14:29:35,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175846 2024-11-13T14:29:35,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175846 2024-11-13T14:29:35,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175846) roll requested 2024-11-13T14:29:35,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175851 2024-11-13T14:29:35,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175851 2024-11-13T14:29:35,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175851) roll requested 2024-11-13T14:29:35,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175857 2024-11-13T14:29:35,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175857 2024-11-13T14:29:35,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175857) roll requested 2024-11-13T14:29:35,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175862 2024-11-13T14:29:35,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175862 2024-11-13T14:29:35,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175862) roll requested 2024-11-13T14:29:35,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175867 2024-11-13T14:29:35,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175867 2024-11-13T14:29:35,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175867) roll requested 2024-11-13T14:29:35,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175872 2024-11-13T14:29:35,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175872 2024-11-13T14:29:35,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175872) roll requested 2024-11-13T14:29:35,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175878 2024-11-13T14:29:35,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175878 2024-11-13T14:29:35,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175878) roll requested 2024-11-13T14:29:35,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175884 2024-11-13T14:29:35,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175884 2024-11-13T14:29:35,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175884) roll requested 2024-11-13T14:29:35,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175892 2024-11-13T14:29:35,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175892 2024-11-13T14:29:35,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175892) roll requested 2024-11-13T14:29:35,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175898 2024-11-13T14:29:35,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175898 2024-11-13T14:29:35,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175898) roll requested 2024-11-13T14:29:35,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175903 2024-11-13T14:29:35,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175903 2024-11-13T14:29:35,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175903) roll requested 2024-11-13T14:29:35,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175908 2024-11-13T14:29:35,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175908 2024-11-13T14:29:35,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175908) roll requested 2024-11-13T14:29:35,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175913 2024-11-13T14:29:35,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175913 2024-11-13T14:29:35,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175913) roll requested 2024-11-13T14:29:35,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175918 2024-11-13T14:29:35,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175918 2024-11-13T14:29:35,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175918) roll requested 2024-11-13T14:29:35,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175923 2024-11-13T14:29:35,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175923 2024-11-13T14:29:35,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175923) roll requested 2024-11-13T14:29:35,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175929 2024-11-13T14:29:35,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175929 2024-11-13T14:29:35,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175929) roll requested 2024-11-13T14:29:35,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175934 2024-11-13T14:29:35,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175934 2024-11-13T14:29:35,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175934) roll requested 2024-11-13T14:29:35,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175940 2024-11-13T14:29:35,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175940 2024-11-13T14:29:35,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175940) roll requested 2024-11-13T14:29:35,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175946 2024-11-13T14:29:35,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175946 2024-11-13T14:29:35,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175946) roll requested 2024-11-13T14:29:35,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175951 2024-11-13T14:29:35,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175951 2024-11-13T14:29:35,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175951) roll requested 2024-11-13T14:29:35,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175957 2024-11-13T14:29:35,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175957 2024-11-13T14:29:35,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175957) roll requested 2024-11-13T14:29:35,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175964 2024-11-13T14:29:35,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175964 2024-11-13T14:29:35,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175964) roll requested 2024-11-13T14:29:35,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175970 2024-11-13T14:29:35,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175970 2024-11-13T14:29:35,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175970) roll requested 2024-11-13T14:29:35,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175975 2024-11-13T14:29:35,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175975 2024-11-13T14:29:35,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175975) roll requested 2024-11-13T14:29:35,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175981 2024-11-13T14:29:35,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175981 2024-11-13T14:29:35,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175981) roll requested 2024-11-13T14:29:35,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175986 2024-11-13T14:29:35,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175986 2024-11-13T14:29:35,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175986) roll requested 2024-11-13T14:29:35,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175991 2024-11-13T14:29:35,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:35,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175991 2024-11-13T14:29:35,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:35,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:35,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175991) roll requested 2024-11-13T14:29:35,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508175997 2024-11-13T14:29:36,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175997 2024-11-13T14:29:36,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508175997) roll requested 2024-11-13T14:29:36,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176003 2024-11-13T14:29:36,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508175997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176003 2024-11-13T14:29:36,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176003) roll requested 2024-11-13T14:29:36,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176007 2024-11-13T14:29:36,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176007 2024-11-13T14:29:36,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176007) roll requested 2024-11-13T14:29:36,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176012 2024-11-13T14:29:36,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176012 2024-11-13T14:29:36,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176012) roll requested 2024-11-13T14:29:36,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176017 2024-11-13T14:29:36,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176017 2024-11-13T14:29:36,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176017) roll requested 2024-11-13T14:29:36,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176023 2024-11-13T14:29:36,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176023 2024-11-13T14:29:36,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176023) roll requested 2024-11-13T14:29:36,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176033 2024-11-13T14:29:36,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176033 2024-11-13T14:29:36,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176033) roll requested 2024-11-13T14:29:36,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176040 2024-11-13T14:29:36,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176040 2024-11-13T14:29:36,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176040) roll requested 2024-11-13T14:29:36,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176047 2024-11-13T14:29:36,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176047 2024-11-13T14:29:36,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176047) roll requested 2024-11-13T14:29:36,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176054 2024-11-13T14:29:36,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176054 2024-11-13T14:29:36,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176054) roll requested 2024-11-13T14:29:36,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176062 2024-11-13T14:29:36,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176062 2024-11-13T14:29:36,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176062) roll requested 2024-11-13T14:29:36,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176070 2024-11-13T14:29:36,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176070 2024-11-13T14:29:36,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176070) roll requested 2024-11-13T14:29:36,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176080 2024-11-13T14:29:36,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176080 2024-11-13T14:29:36,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176080) roll requested 2024-11-13T14:29:36,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176090 2024-11-13T14:29:36,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176090 2024-11-13T14:29:36,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176090) roll requested 2024-11-13T14:29:36,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176097 2024-11-13T14:29:36,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176097 2024-11-13T14:29:36,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176097) roll requested 2024-11-13T14:29:36,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176104 2024-11-13T14:29:36,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176104 2024-11-13T14:29:36,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176104) roll requested 2024-11-13T14:29:36,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176111 2024-11-13T14:29:36,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176111 2024-11-13T14:29:36,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176111) roll requested 2024-11-13T14:29:36,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176117 2024-11-13T14:29:36,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176117 2024-11-13T14:29:36,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176117) roll requested 2024-11-13T14:29:36,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176122 2024-11-13T14:29:36,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176122 2024-11-13T14:29:36,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176122) roll requested 2024-11-13T14:29:36,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176128 2024-11-13T14:29:36,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176128 2024-11-13T14:29:36,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176128) roll requested 2024-11-13T14:29:36,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176135 2024-11-13T14:29:36,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176135 2024-11-13T14:29:36,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176135) roll requested 2024-11-13T14:29:36,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176143 2024-11-13T14:29:36,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176143 2024-11-13T14:29:36,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176143) roll requested 2024-11-13T14:29:36,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176152 2024-11-13T14:29:36,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176152 2024-11-13T14:29:36,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176152) roll requested 2024-11-13T14:29:36,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176160 2024-11-13T14:29:36,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176160 2024-11-13T14:29:36,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176160) roll requested 2024-11-13T14:29:36,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176169 2024-11-13T14:29:36,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176169 2024-11-13T14:29:36,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176169) roll requested 2024-11-13T14:29:36,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176177 2024-11-13T14:29:36,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176177 2024-11-13T14:29:36,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176177) roll requested 2024-11-13T14:29:36,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176184 2024-11-13T14:29:36,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176184 2024-11-13T14:29:36,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176184) roll requested 2024-11-13T14:29:36,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176192 2024-11-13T14:29:36,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176192 2024-11-13T14:29:36,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176192) roll requested 2024-11-13T14:29:36,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176201 2024-11-13T14:29:36,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176201 2024-11-13T14:29:36,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176201) roll requested 2024-11-13T14:29:36,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176207 2024-11-13T14:29:36,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176207 2024-11-13T14:29:36,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176207) roll requested 2024-11-13T14:29:36,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176214 2024-11-13T14:29:36,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176214 2024-11-13T14:29:36,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176214) roll requested 2024-11-13T14:29:36,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176219 2024-11-13T14:29:36,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176219 2024-11-13T14:29:36,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176219) roll requested 2024-11-13T14:29:36,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176224 2024-11-13T14:29:36,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176224 2024-11-13T14:29:36,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176224) roll requested 2024-11-13T14:29:36,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176229 2024-11-13T14:29:36,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176229 2024-11-13T14:29:36,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176229) roll requested 2024-11-13T14:29:36,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176234 2024-11-13T14:29:36,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176234 2024-11-13T14:29:36,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176234) roll requested 2024-11-13T14:29:36,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176239 2024-11-13T14:29:36,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176239 2024-11-13T14:29:36,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176239) roll requested 2024-11-13T14:29:36,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176245 2024-11-13T14:29:36,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176245 2024-11-13T14:29:36,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176245) roll requested 2024-11-13T14:29:36,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176251 2024-11-13T14:29:36,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176251 2024-11-13T14:29:36,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176251) roll requested 2024-11-13T14:29:36,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176256 2024-11-13T14:29:36,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176256 2024-11-13T14:29:36,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176256) roll requested 2024-11-13T14:29:36,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176261 2024-11-13T14:29:36,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176261 2024-11-13T14:29:36,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176261) roll requested 2024-11-13T14:29:36,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176268 2024-11-13T14:29:36,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176268 2024-11-13T14:29:36,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176268) roll requested 2024-11-13T14:29:36,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176276 2024-11-13T14:29:36,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176276 2024-11-13T14:29:36,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176276) roll requested 2024-11-13T14:29:36,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176284 2024-11-13T14:29:36,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176284 2024-11-13T14:29:36,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176284) roll requested 2024-11-13T14:29:36,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176290 2024-11-13T14:29:36,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176290 2024-11-13T14:29:36,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176290) roll requested 2024-11-13T14:29:36,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176296 2024-11-13T14:29:36,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176296 2024-11-13T14:29:36,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176296) roll requested 2024-11-13T14:29:36,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176301 2024-11-13T14:29:36,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176301 2024-11-13T14:29:36,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176301) roll requested 2024-11-13T14:29:36,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176307 2024-11-13T14:29:36,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176307 2024-11-13T14:29:36,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176307) roll requested 2024-11-13T14:29:36,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176314 2024-11-13T14:29:36,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176314 2024-11-13T14:29:36,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176314) roll requested 2024-11-13T14:29:36,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176321 2024-11-13T14:29:36,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176321 2024-11-13T14:29:36,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176321) roll requested 2024-11-13T14:29:36,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176329 2024-11-13T14:29:36,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176329 2024-11-13T14:29:36,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176329) roll requested 2024-11-13T14:29:36,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176339 2024-11-13T14:29:36,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176339 2024-11-13T14:29:36,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176339) roll requested 2024-11-13T14:29:36,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176348 2024-11-13T14:29:36,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176348 2024-11-13T14:29:36,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176348) roll requested 2024-11-13T14:29:36,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176354 2024-11-13T14:29:36,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176354 2024-11-13T14:29:36,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176354) roll requested 2024-11-13T14:29:36,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176363 2024-11-13T14:29:36,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176363 2024-11-13T14:29:36,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176363) roll requested 2024-11-13T14:29:36,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176368 2024-11-13T14:29:36,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176368 2024-11-13T14:29:36,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176368) roll requested 2024-11-13T14:29:36,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176376 2024-11-13T14:29:36,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176376 2024-11-13T14:29:36,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176376) roll requested 2024-11-13T14:29:36,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176382 2024-11-13T14:29:36,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176382 2024-11-13T14:29:36,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176382) roll requested 2024-11-13T14:29:36,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176387 2024-11-13T14:29:36,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176387 2024-11-13T14:29:36,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176387) roll requested 2024-11-13T14:29:36,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176393 2024-11-13T14:29:36,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176393 2024-11-13T14:29:36,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176393) roll requested 2024-11-13T14:29:36,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176399 2024-11-13T14:29:36,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176399 2024-11-13T14:29:36,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176399) roll requested 2024-11-13T14:29:36,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176404 2024-11-13T14:29:36,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176404 2024-11-13T14:29:36,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176404) roll requested 2024-11-13T14:29:36,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176412 2024-11-13T14:29:36,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176412 2024-11-13T14:29:36,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176412) roll requested 2024-11-13T14:29:36,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176417 2024-11-13T14:29:36,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176417 2024-11-13T14:29:36,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176417) roll requested 2024-11-13T14:29:36,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176422 2024-11-13T14:29:36,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176422 2024-11-13T14:29:36,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176422) roll requested 2024-11-13T14:29:36,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176428 2024-11-13T14:29:36,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176428 2024-11-13T14:29:36,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176428) roll requested 2024-11-13T14:29:36,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176433 2024-11-13T14:29:36,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176433 2024-11-13T14:29:36,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176433) roll requested 2024-11-13T14:29:36,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176439 2024-11-13T14:29:36,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176439 2024-11-13T14:29:36,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176439) roll requested 2024-11-13T14:29:36,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176445 2024-11-13T14:29:36,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176445 2024-11-13T14:29:36,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176445) roll requested 2024-11-13T14:29:36,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176450 2024-11-13T14:29:36,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176450 2024-11-13T14:29:36,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176450) roll requested 2024-11-13T14:29:36,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176455 2024-11-13T14:29:36,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176455 2024-11-13T14:29:36,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176455) roll requested 2024-11-13T14:29:36,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176462 2024-11-13T14:29:36,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176462 2024-11-13T14:29:36,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176462) roll requested 2024-11-13T14:29:36,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176467 2024-11-13T14:29:36,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176467 2024-11-13T14:29:36,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176467) roll requested 2024-11-13T14:29:36,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176473 2024-11-13T14:29:36,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176473 2024-11-13T14:29:36,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176473) roll requested 2024-11-13T14:29:36,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176479 2024-11-13T14:29:36,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176479 2024-11-13T14:29:36,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176479) roll requested 2024-11-13T14:29:36,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176485 2024-11-13T14:29:36,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176485 2024-11-13T14:29:36,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176485) roll requested 2024-11-13T14:29:36,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176493 2024-11-13T14:29:36,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176493 2024-11-13T14:29:36,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176493) roll requested 2024-11-13T14:29:36,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176500 2024-11-13T14:29:36,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176500 2024-11-13T14:29:36,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176500) roll requested 2024-11-13T14:29:36,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176506 2024-11-13T14:29:36,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176506 2024-11-13T14:29:36,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176506) roll requested 2024-11-13T14:29:36,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176514 2024-11-13T14:29:36,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176514 2024-11-13T14:29:36,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176514) roll requested 2024-11-13T14:29:36,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176524 2024-11-13T14:29:36,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176524 2024-11-13T14:29:36,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176524) roll requested 2024-11-13T14:29:36,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176532 2024-11-13T14:29:36,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176532 2024-11-13T14:29:36,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176532) roll requested 2024-11-13T14:29:36,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176538 2024-11-13T14:29:36,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176538 2024-11-13T14:29:36,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176538) roll requested 2024-11-13T14:29:36,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176546 2024-11-13T14:29:36,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176546 2024-11-13T14:29:36,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176546) roll requested 2024-11-13T14:29:36,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176555 2024-11-13T14:29:36,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176555 2024-11-13T14:29:36,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176555) roll requested 2024-11-13T14:29:36,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176564 2024-11-13T14:29:36,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176564 2024-11-13T14:29:36,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176564) roll requested 2024-11-13T14:29:36,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176573 2024-11-13T14:29:36,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176573 2024-11-13T14:29:36,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176573) roll requested 2024-11-13T14:29:36,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176582 2024-11-13T14:29:36,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176582 2024-11-13T14:29:36,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176582) roll requested 2024-11-13T14:29:36,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176594 2024-11-13T14:29:36,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176594 2024-11-13T14:29:36,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176594) roll requested 2024-11-13T14:29:36,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176603 2024-11-13T14:29:36,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176603 2024-11-13T14:29:36,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176603) roll requested 2024-11-13T14:29:36,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176612 2024-11-13T14:29:36,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176612 2024-11-13T14:29:36,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176612) roll requested 2024-11-13T14:29:36,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176620 2024-11-13T14:29:36,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176620 2024-11-13T14:29:36,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176620) roll requested 2024-11-13T14:29:36,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176625 2024-11-13T14:29:36,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176625 2024-11-13T14:29:36,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176625) roll requested 2024-11-13T14:29:36,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176633 2024-11-13T14:29:36,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176633 2024-11-13T14:29:36,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176633) roll requested 2024-11-13T14:29:36,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176641 2024-11-13T14:29:36,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176641 2024-11-13T14:29:36,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176641) roll requested 2024-11-13T14:29:36,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176648 2024-11-13T14:29:36,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176648 2024-11-13T14:29:36,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176648) roll requested 2024-11-13T14:29:36,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176657 2024-11-13T14:29:36,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176657 2024-11-13T14:29:36,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176657) roll requested 2024-11-13T14:29:36,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176666 2024-11-13T14:29:36,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176666 2024-11-13T14:29:36,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176666) roll requested 2024-11-13T14:29:36,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176676 2024-11-13T14:29:36,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176676 2024-11-13T14:29:36,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176676) roll requested 2024-11-13T14:29:36,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176684 2024-11-13T14:29:36,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176684 2024-11-13T14:29:36,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176684) roll requested 2024-11-13T14:29:36,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176700 2024-11-13T14:29:36,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176700 2024-11-13T14:29:36,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176700) roll requested 2024-11-13T14:29:36,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176705 2024-11-13T14:29:36,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176705 2024-11-13T14:29:36,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176705) roll requested 2024-11-13T14:29:36,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176711 2024-11-13T14:29:36,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176711 2024-11-13T14:29:36,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176711) roll requested 2024-11-13T14:29:36,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176718 2024-11-13T14:29:36,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176718 2024-11-13T14:29:36,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176718) roll requested 2024-11-13T14:29:36,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176724 2024-11-13T14:29:36,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176724 2024-11-13T14:29:36,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176724) roll requested 2024-11-13T14:29:36,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176732 2024-11-13T14:29:36,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176732 2024-11-13T14:29:36,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176732) roll requested 2024-11-13T14:29:36,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176739 2024-11-13T14:29:36,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176739 2024-11-13T14:29:36,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176739) roll requested 2024-11-13T14:29:36,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176748 2024-11-13T14:29:36,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176748 2024-11-13T14:29:36,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176748) roll requested 2024-11-13T14:29:36,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176757 2024-11-13T14:29:36,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176757 2024-11-13T14:29:36,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176757) roll requested 2024-11-13T14:29:36,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176763 2024-11-13T14:29:36,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176763 2024-11-13T14:29:36,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176763) roll requested 2024-11-13T14:29:36,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176773 2024-11-13T14:29:36,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176773 2024-11-13T14:29:36,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176773) roll requested 2024-11-13T14:29:36,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176781 2024-11-13T14:29:36,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176781 2024-11-13T14:29:36,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176781) roll requested 2024-11-13T14:29:36,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176790 2024-11-13T14:29:36,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176790 2024-11-13T14:29:36,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176790) roll requested 2024-11-13T14:29:36,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176800 2024-11-13T14:29:36,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176800 2024-11-13T14:29:36,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176800) roll requested 2024-11-13T14:29:36,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176808 2024-11-13T14:29:36,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176808 2024-11-13T14:29:36,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176808) roll requested 2024-11-13T14:29:36,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176815 2024-11-13T14:29:36,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176815 2024-11-13T14:29:36,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176815) roll requested 2024-11-13T14:29:36,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176823 2024-11-13T14:29:36,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176823 2024-11-13T14:29:36,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176823) roll requested 2024-11-13T14:29:36,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176832 2024-11-13T14:29:36,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176832 2024-11-13T14:29:36,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176832) roll requested 2024-11-13T14:29:36,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176841 2024-11-13T14:29:36,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176841 2024-11-13T14:29:36,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176841) roll requested 2024-11-13T14:29:36,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176850 2024-11-13T14:29:36,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176850 2024-11-13T14:29:36,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176850) roll requested 2024-11-13T14:29:36,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176858 2024-11-13T14:29:36,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176858 2024-11-13T14:29:36,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176858) roll requested 2024-11-13T14:29:36,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176865 2024-11-13T14:29:36,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176865 2024-11-13T14:29:36,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176865) roll requested 2024-11-13T14:29:36,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176874 2024-11-13T14:29:36,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176874 2024-11-13T14:29:36,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176874) roll requested 2024-11-13T14:29:36,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176882 2024-11-13T14:29:36,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176882 2024-11-13T14:29:36,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176882) roll requested 2024-11-13T14:29:36,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176892 2024-11-13T14:29:36,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176892 2024-11-13T14:29:36,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176892) roll requested 2024-11-13T14:29:36,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176901 2024-11-13T14:29:36,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176901 2024-11-13T14:29:36,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176901) roll requested 2024-11-13T14:29:36,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176906 2024-11-13T14:29:36,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176906 2024-11-13T14:29:36,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176906) roll requested 2024-11-13T14:29:36,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176911 2024-11-13T14:29:36,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176911 2024-11-13T14:29:36,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176911) roll requested 2024-11-13T14:29:36,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176917 2024-11-13T14:29:36,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176917 2024-11-13T14:29:36,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176917) roll requested 2024-11-13T14:29:36,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176925 2024-11-13T14:29:36,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176925 2024-11-13T14:29:36,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176925) roll requested 2024-11-13T14:29:36,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176934 2024-11-13T14:29:36,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176934 2024-11-13T14:29:36,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176934) roll requested 2024-11-13T14:29:36,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176944 2024-11-13T14:29:36,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176944 2024-11-13T14:29:36,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176944) roll requested 2024-11-13T14:29:36,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176952 2024-11-13T14:29:36,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176952 2024-11-13T14:29:36,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176952) roll requested 2024-11-13T14:29:36,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176960 2024-11-13T14:29:36,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176960 2024-11-13T14:29:36,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176960) roll requested 2024-11-13T14:29:36,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176969 2024-11-13T14:29:36,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176969 2024-11-13T14:29:36,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176969) roll requested 2024-11-13T14:29:36,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176978 2024-11-13T14:29:36,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176978 2024-11-13T14:29:36,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176978) roll requested 2024-11-13T14:29:36,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176985 2024-11-13T14:29:36,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176985 2024-11-13T14:29:36,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:36,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:36,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176985) roll requested 2024-11-13T14:29:36,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508176993 2024-11-13T14:29:36,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:36,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176993 2024-11-13T14:29:37,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508176993) roll requested 2024-11-13T14:29:37,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177000 2024-11-13T14:29:37,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508176993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177000 2024-11-13T14:29:37,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177000) roll requested 2024-11-13T14:29:37,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177007 2024-11-13T14:29:37,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177007 2024-11-13T14:29:37,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177007) roll requested 2024-11-13T14:29:37,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177014 2024-11-13T14:29:37,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177014 2024-11-13T14:29:37,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177014) roll requested 2024-11-13T14:29:37,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177020 2024-11-13T14:29:37,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177020 2024-11-13T14:29:37,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177020) roll requested 2024-11-13T14:29:37,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177027 2024-11-13T14:29:37,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177027 2024-11-13T14:29:37,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177027) roll requested 2024-11-13T14:29:37,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177034 2024-11-13T14:29:37,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177034 2024-11-13T14:29:37,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177034) roll requested 2024-11-13T14:29:37,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177042 2024-11-13T14:29:37,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177042 2024-11-13T14:29:37,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177042) roll requested 2024-11-13T14:29:37,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177049 2024-11-13T14:29:37,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177049 2024-11-13T14:29:37,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177049) roll requested 2024-11-13T14:29:37,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177057 2024-11-13T14:29:37,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177057 2024-11-13T14:29:37,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177057) roll requested 2024-11-13T14:29:37,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177066 2024-11-13T14:29:37,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177066 2024-11-13T14:29:37,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177066) roll requested 2024-11-13T14:29:37,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177075 2024-11-13T14:29:37,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177075 2024-11-13T14:29:37,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177075) roll requested 2024-11-13T14:29:37,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177082 2024-11-13T14:29:37,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177082 2024-11-13T14:29:37,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177082) roll requested 2024-11-13T14:29:37,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177091 2024-11-13T14:29:37,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177091 2024-11-13T14:29:37,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177091) roll requested 2024-11-13T14:29:37,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177100 2024-11-13T14:29:37,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177100 2024-11-13T14:29:37,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177100) roll requested 2024-11-13T14:29:37,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177110 2024-11-13T14:29:37,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177110 2024-11-13T14:29:37,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177110) roll requested 2024-11-13T14:29:37,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177119 2024-11-13T14:29:37,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177119 2024-11-13T14:29:37,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177119) roll requested 2024-11-13T14:29:37,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177128 2024-11-13T14:29:37,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177128 2024-11-13T14:29:37,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177128) roll requested 2024-11-13T14:29:37,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177135 2024-11-13T14:29:37,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177135 2024-11-13T14:29:37,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177135) roll requested 2024-11-13T14:29:37,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177143 2024-11-13T14:29:37,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177143 2024-11-13T14:29:37,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177143) roll requested 2024-11-13T14:29:37,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177152 2024-11-13T14:29:37,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177152 2024-11-13T14:29:37,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177152) roll requested 2024-11-13T14:29:37,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177160 2024-11-13T14:29:37,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177160 2024-11-13T14:29:37,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177160) roll requested 2024-11-13T14:29:37,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177167 2024-11-13T14:29:37,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177167 2024-11-13T14:29:37,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177167) roll requested 2024-11-13T14:29:37,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177174 2024-11-13T14:29:37,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177174 2024-11-13T14:29:37,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177174) roll requested 2024-11-13T14:29:37,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177183 2024-11-13T14:29:37,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177183 2024-11-13T14:29:37,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177183) roll requested 2024-11-13T14:29:37,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177192 2024-11-13T14:29:37,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177192 2024-11-13T14:29:37,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177192) roll requested 2024-11-13T14:29:37,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177201 2024-11-13T14:29:37,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177201 2024-11-13T14:29:37,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177201) roll requested 2024-11-13T14:29:37,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177210 2024-11-13T14:29:37,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177210 2024-11-13T14:29:37,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177210) roll requested 2024-11-13T14:29:37,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177219 2024-11-13T14:29:37,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177219 2024-11-13T14:29:37,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177219) roll requested 2024-11-13T14:29:37,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177226 2024-11-13T14:29:37,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177226 2024-11-13T14:29:37,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177226) roll requested 2024-11-13T14:29:37,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177234 2024-11-13T14:29:37,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177234 2024-11-13T14:29:37,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177234) roll requested 2024-11-13T14:29:37,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177241 2024-11-13T14:29:37,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177241 2024-11-13T14:29:37,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177241) roll requested 2024-11-13T14:29:37,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177249 2024-11-13T14:29:37,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177249 2024-11-13T14:29:37,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177249) roll requested 2024-11-13T14:29:37,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177257 2024-11-13T14:29:37,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177257 2024-11-13T14:29:37,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177257) roll requested 2024-11-13T14:29:37,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177264 2024-11-13T14:29:37,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177264 2024-11-13T14:29:37,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177264) roll requested 2024-11-13T14:29:37,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177270 2024-11-13T14:29:37,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177270 2024-11-13T14:29:37,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177270) roll requested 2024-11-13T14:29:37,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177277 2024-11-13T14:29:37,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177277 2024-11-13T14:29:37,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177277) roll requested 2024-11-13T14:29:37,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177284 2024-11-13T14:29:37,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177284 2024-11-13T14:29:37,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177284) roll requested 2024-11-13T14:29:37,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177293 2024-11-13T14:29:37,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177293 2024-11-13T14:29:37,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177293) roll requested 2024-11-13T14:29:37,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177302 2024-11-13T14:29:37,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177302 2024-11-13T14:29:37,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177302) roll requested 2024-11-13T14:29:37,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177311 2024-11-13T14:29:37,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177311 2024-11-13T14:29:37,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177311) roll requested 2024-11-13T14:29:37,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177321 2024-11-13T14:29:37,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177321 2024-11-13T14:29:37,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177321) roll requested 2024-11-13T14:29:37,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177329 2024-11-13T14:29:37,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177329 2024-11-13T14:29:37,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177329) roll requested 2024-11-13T14:29:37,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177338 2024-11-13T14:29:37,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177338 2024-11-13T14:29:37,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177338) roll requested 2024-11-13T14:29:37,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177347 2024-11-13T14:29:37,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177347 2024-11-13T14:29:37,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177347) roll requested 2024-11-13T14:29:37,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177357 2024-11-13T14:29:37,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177357 2024-11-13T14:29:37,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177357) roll requested 2024-11-13T14:29:37,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177367 2024-11-13T14:29:37,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177367 2024-11-13T14:29:37,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177367) roll requested 2024-11-13T14:29:37,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177378 2024-11-13T14:29:37,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177378 2024-11-13T14:29:37,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177378) roll requested 2024-11-13T14:29:37,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177383 2024-11-13T14:29:37,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177383 2024-11-13T14:29:37,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177383) roll requested 2024-11-13T14:29:37,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177388 2024-11-13T14:29:37,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177388 2024-11-13T14:29:37,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177388) roll requested 2024-11-13T14:29:37,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177393 2024-11-13T14:29:37,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177393 2024-11-13T14:29:37,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177393) roll requested 2024-11-13T14:29:37,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177400 2024-11-13T14:29:37,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177400 2024-11-13T14:29:37,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177400) roll requested 2024-11-13T14:29:37,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177406 2024-11-13T14:29:37,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177406 2024-11-13T14:29:37,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177406) roll requested 2024-11-13T14:29:37,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177412 2024-11-13T14:29:37,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177412 2024-11-13T14:29:37,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177412) roll requested 2024-11-13T14:29:37,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177420 2024-11-13T14:29:37,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177420 2024-11-13T14:29:37,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177420) roll requested 2024-11-13T14:29:37,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177427 2024-11-13T14:29:37,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177427 2024-11-13T14:29:37,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177427) roll requested 2024-11-13T14:29:37,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177433 2024-11-13T14:29:37,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177433 2024-11-13T14:29:37,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177433) roll requested 2024-11-13T14:29:37,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177440 2024-11-13T14:29:37,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177440 2024-11-13T14:29:37,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177440) roll requested 2024-11-13T14:29:37,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177447 2024-11-13T14:29:37,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177447 2024-11-13T14:29:37,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177447) roll requested 2024-11-13T14:29:37,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177456 2024-11-13T14:29:37,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177456 2024-11-13T14:29:37,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177456) roll requested 2024-11-13T14:29:37,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177465 2024-11-13T14:29:37,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177465 2024-11-13T14:29:37,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177465) roll requested 2024-11-13T14:29:37,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177471 2024-11-13T14:29:37,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177471 2024-11-13T14:29:37,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177471) roll requested 2024-11-13T14:29:37,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177480 2024-11-13T14:29:37,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177480 2024-11-13T14:29:37,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177480) roll requested 2024-11-13T14:29:37,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177488 2024-11-13T14:29:37,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177488 2024-11-13T14:29:37,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177488) roll requested 2024-11-13T14:29:37,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177495 2024-11-13T14:29:37,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177495 2024-11-13T14:29:37,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177495) roll requested 2024-11-13T14:29:37,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177505 2024-11-13T14:29:37,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177505 2024-11-13T14:29:37,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177505) roll requested 2024-11-13T14:29:37,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177511 2024-11-13T14:29:37,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177511 2024-11-13T14:29:37,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177511) roll requested 2024-11-13T14:29:37,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177518 2024-11-13T14:29:37,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177518 2024-11-13T14:29:37,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177518) roll requested 2024-11-13T14:29:37,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177527 2024-11-13T14:29:37,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177527 2024-11-13T14:29:37,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177527) roll requested 2024-11-13T14:29:37,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177537 2024-11-13T14:29:37,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177537 2024-11-13T14:29:37,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177537) roll requested 2024-11-13T14:29:37,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177543 2024-11-13T14:29:37,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177543 2024-11-13T14:29:37,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177543) roll requested 2024-11-13T14:29:37,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177550 2024-11-13T14:29:37,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177550 2024-11-13T14:29:37,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177550) roll requested 2024-11-13T14:29:37,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177557 2024-11-13T14:29:37,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177557 2024-11-13T14:29:37,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177557) roll requested 2024-11-13T14:29:37,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177564 2024-11-13T14:29:37,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177564 2024-11-13T14:29:37,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177564) roll requested 2024-11-13T14:29:37,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177572 2024-11-13T14:29:37,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177572 2024-11-13T14:29:37,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177572) roll requested 2024-11-13T14:29:37,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177583 2024-11-13T14:29:37,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177583 2024-11-13T14:29:37,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177583) roll requested 2024-11-13T14:29:37,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177590 2024-11-13T14:29:37,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177590 2024-11-13T14:29:37,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177590) roll requested 2024-11-13T14:29:37,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177599 2024-11-13T14:29:37,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177599 2024-11-13T14:29:37,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177599) roll requested 2024-11-13T14:29:37,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177609 2024-11-13T14:29:37,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177609 2024-11-13T14:29:37,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177609) roll requested 2024-11-13T14:29:37,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177617 2024-11-13T14:29:37,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177617 2024-11-13T14:29:37,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177617) roll requested 2024-11-13T14:29:37,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177624 2024-11-13T14:29:37,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177624 2024-11-13T14:29:37,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177624) roll requested 2024-11-13T14:29:37,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177632 2024-11-13T14:29:37,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177632 2024-11-13T14:29:37,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177632) roll requested 2024-11-13T14:29:37,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177639 2024-11-13T14:29:37,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177639 2024-11-13T14:29:37,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177639) roll requested 2024-11-13T14:29:37,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177645 2024-11-13T14:29:37,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177645 2024-11-13T14:29:37,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177645) roll requested 2024-11-13T14:29:37,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177653 2024-11-13T14:29:37,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177653 2024-11-13T14:29:37,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177653) roll requested 2024-11-13T14:29:37,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177663 2024-11-13T14:29:37,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177663 2024-11-13T14:29:37,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177663) roll requested 2024-11-13T14:29:37,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177670 2024-11-13T14:29:37,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177670 2024-11-13T14:29:37,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177670) roll requested 2024-11-13T14:29:37,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177678 2024-11-13T14:29:37,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177678 2024-11-13T14:29:37,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177678) roll requested 2024-11-13T14:29:37,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177687 2024-11-13T14:29:37,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177687 2024-11-13T14:29:37,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177687) roll requested 2024-11-13T14:29:37,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177697 2024-11-13T14:29:37,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177697 2024-11-13T14:29:37,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177697) roll requested 2024-11-13T14:29:37,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177706 2024-11-13T14:29:37,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177706 2024-11-13T14:29:37,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177706) roll requested 2024-11-13T14:29:37,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177713 2024-11-13T14:29:37,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177713 2024-11-13T14:29:37,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177713) roll requested 2024-11-13T14:29:37,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177722 2024-11-13T14:29:37,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177722 2024-11-13T14:29:37,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177722) roll requested 2024-11-13T14:29:37,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177730 2024-11-13T14:29:37,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177730 2024-11-13T14:29:37,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177730) roll requested 2024-11-13T14:29:37,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177736 2024-11-13T14:29:37,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177736 2024-11-13T14:29:37,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177736) roll requested 2024-11-13T14:29:37,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177742 2024-11-13T14:29:37,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177742 2024-11-13T14:29:37,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177742) roll requested 2024-11-13T14:29:37,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177750 2024-11-13T14:29:37,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177750 2024-11-13T14:29:37,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177750) roll requested 2024-11-13T14:29:37,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177758 2024-11-13T14:29:37,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177758 2024-11-13T14:29:37,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177758) roll requested 2024-11-13T14:29:37,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177765 2024-11-13T14:29:37,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177765 2024-11-13T14:29:37,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177765) roll requested 2024-11-13T14:29:37,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177772 2024-11-13T14:29:37,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177772 2024-11-13T14:29:37,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177772) roll requested 2024-11-13T14:29:37,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177780 2024-11-13T14:29:37,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177780 2024-11-13T14:29:37,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177780) roll requested 2024-11-13T14:29:37,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177788 2024-11-13T14:29:37,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177788 2024-11-13T14:29:37,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177788) roll requested 2024-11-13T14:29:37,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177796 2024-11-13T14:29:37,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177796 2024-11-13T14:29:37,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177796) roll requested 2024-11-13T14:29:37,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177805 2024-11-13T14:29:37,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177805 2024-11-13T14:29:37,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177805) roll requested 2024-11-13T14:29:37,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177813 2024-11-13T14:29:37,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177813 2024-11-13T14:29:37,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177813) roll requested 2024-11-13T14:29:37,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177822 2024-11-13T14:29:37,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177822 2024-11-13T14:29:37,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177822) roll requested 2024-11-13T14:29:37,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177830 2024-11-13T14:29:37,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177830 2024-11-13T14:29:37,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177830) roll requested 2024-11-13T14:29:37,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177840 2024-11-13T14:29:37,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177840 2024-11-13T14:29:37,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177840) roll requested 2024-11-13T14:29:37,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177847 2024-11-13T14:29:37,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177847 2024-11-13T14:29:37,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177847) roll requested 2024-11-13T14:29:37,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177855 2024-11-13T14:29:37,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177855 2024-11-13T14:29:37,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177855) roll requested 2024-11-13T14:29:37,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177863 2024-11-13T14:29:37,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177863 2024-11-13T14:29:37,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177863) roll requested 2024-11-13T14:29:37,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177873 2024-11-13T14:29:37,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177873 2024-11-13T14:29:37,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177873) roll requested 2024-11-13T14:29:37,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177883 2024-11-13T14:29:37,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177883 2024-11-13T14:29:37,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177883) roll requested 2024-11-13T14:29:37,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177892 2024-11-13T14:29:37,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177892 2024-11-13T14:29:37,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177892) roll requested 2024-11-13T14:29:37,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177901 2024-11-13T14:29:37,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177901 2024-11-13T14:29:37,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177901) roll requested 2024-11-13T14:29:37,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177908 2024-11-13T14:29:37,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177908 2024-11-13T14:29:37,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177908) roll requested 2024-11-13T14:29:37,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177917 2024-11-13T14:29:37,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177917 2024-11-13T14:29:37,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177917) roll requested 2024-11-13T14:29:37,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177926 2024-11-13T14:29:37,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177926 2024-11-13T14:29:37,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177926) roll requested 2024-11-13T14:29:37,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177935 2024-11-13T14:29:37,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177935 2024-11-13T14:29:37,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177935) roll requested 2024-11-13T14:29:37,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177944 2024-11-13T14:29:37,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177944 2024-11-13T14:29:37,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177944) roll requested 2024-11-13T14:29:37,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177951 2024-11-13T14:29:37,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177951 2024-11-13T14:29:37,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177951) roll requested 2024-11-13T14:29:37,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177964 2024-11-13T14:29:37,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177964 2024-11-13T14:29:37,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177964) roll requested 2024-11-13T14:29:37,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177972 2024-11-13T14:29:37,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177972 2024-11-13T14:29:37,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177972) roll requested 2024-11-13T14:29:37,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177978 2024-11-13T14:29:37,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177978 2024-11-13T14:29:37,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177978) roll requested 2024-11-13T14:29:37,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177983 2024-11-13T14:29:37,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177983 2024-11-13T14:29:37,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177983) roll requested 2024-11-13T14:29:37,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177988 2024-11-13T14:29:37,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:37,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177988 2024-11-13T14:29:37,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:37,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:37,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177988) roll requested 2024-11-13T14:29:37,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508177995 2024-11-13T14:29:38,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177995 2024-11-13T14:29:38,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508177995) roll requested 2024-11-13T14:29:38,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178002 2024-11-13T14:29:38,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508177995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178002 2024-11-13T14:29:38,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178002) roll requested 2024-11-13T14:29:38,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178007 2024-11-13T14:29:38,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178007 2024-11-13T14:29:38,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178007) roll requested 2024-11-13T14:29:38,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178016 2024-11-13T14:29:38,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178016 2024-11-13T14:29:38,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178016) roll requested 2024-11-13T14:29:38,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178024 2024-11-13T14:29:38,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178024 2024-11-13T14:29:38,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178024) roll requested 2024-11-13T14:29:38,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178032 2024-11-13T14:29:38,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178032 2024-11-13T14:29:38,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178032) roll requested 2024-11-13T14:29:38,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178039 2024-11-13T14:29:38,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178039 2024-11-13T14:29:38,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178039) roll requested 2024-11-13T14:29:38,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178047 2024-11-13T14:29:38,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178047 2024-11-13T14:29:38,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178047) roll requested 2024-11-13T14:29:38,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178056 2024-11-13T14:29:38,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178056 2024-11-13T14:29:38,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178056) roll requested 2024-11-13T14:29:38,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178065 2024-11-13T14:29:38,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178065 2024-11-13T14:29:38,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178065) roll requested 2024-11-13T14:29:38,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178074 2024-11-13T14:29:38,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178074 2024-11-13T14:29:38,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178074) roll requested 2024-11-13T14:29:38,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178084 2024-11-13T14:29:38,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178084 2024-11-13T14:29:38,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178084) roll requested 2024-11-13T14:29:38,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178092 2024-11-13T14:29:38,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178092 2024-11-13T14:29:38,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178092) roll requested 2024-11-13T14:29:38,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178099 2024-11-13T14:29:38,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178099 2024-11-13T14:29:38,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178099) roll requested 2024-11-13T14:29:38,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178109 2024-11-13T14:29:38,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178109 2024-11-13T14:29:38,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178109) roll requested 2024-11-13T14:29:38,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178119 2024-11-13T14:29:38,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178119 2024-11-13T14:29:38,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178119) roll requested 2024-11-13T14:29:38,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178128 2024-11-13T14:29:38,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178128 2024-11-13T14:29:38,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178128) roll requested 2024-11-13T14:29:38,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178136 2024-11-13T14:29:38,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178136 2024-11-13T14:29:38,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178136) roll requested 2024-11-13T14:29:38,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178146 2024-11-13T14:29:38,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178146 2024-11-13T14:29:38,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178146) roll requested 2024-11-13T14:29:38,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178152 2024-11-13T14:29:38,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178152 2024-11-13T14:29:38,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178152) roll requested 2024-11-13T14:29:38,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178159 2024-11-13T14:29:38,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178159 2024-11-13T14:29:38,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178159) roll requested 2024-11-13T14:29:38,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178166 2024-11-13T14:29:38,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178166 2024-11-13T14:29:38,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178166) roll requested 2024-11-13T14:29:38,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178174 2024-11-13T14:29:38,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178174 2024-11-13T14:29:38,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178174) roll requested 2024-11-13T14:29:38,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178182 2024-11-13T14:29:38,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178182 2024-11-13T14:29:38,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178182) roll requested 2024-11-13T14:29:38,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178188 2024-11-13T14:29:38,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178188 2024-11-13T14:29:38,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178188) roll requested 2024-11-13T14:29:38,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178194 2024-11-13T14:29:38,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178194 2024-11-13T14:29:38,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178194) roll requested 2024-11-13T14:29:38,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178198 2024-11-13T14:29:38,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178198 2024-11-13T14:29:38,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178198) roll requested 2024-11-13T14:29:38,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178203 2024-11-13T14:29:38,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178203 2024-11-13T14:29:38,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178203) roll requested 2024-11-13T14:29:38,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178208 2024-11-13T14:29:38,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178208 2024-11-13T14:29:38,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178208) roll requested 2024-11-13T14:29:38,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178213 2024-11-13T14:29:38,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178213 2024-11-13T14:29:38,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178213) roll requested 2024-11-13T14:29:38,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178219 2024-11-13T14:29:38,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178219 2024-11-13T14:29:38,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178219) roll requested 2024-11-13T14:29:38,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178226 2024-11-13T14:29:38,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178226 2024-11-13T14:29:38,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178226) roll requested 2024-11-13T14:29:38,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178234 2024-11-13T14:29:38,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178234 2024-11-13T14:29:38,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178234) roll requested 2024-11-13T14:29:38,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178242 2024-11-13T14:29:38,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178242 2024-11-13T14:29:38,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178242) roll requested 2024-11-13T14:29:38,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178252 2024-11-13T14:29:38,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178252 2024-11-13T14:29:38,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178252) roll requested 2024-11-13T14:29:38,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178260 2024-11-13T14:29:38,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178260 2024-11-13T14:29:38,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178260) roll requested 2024-11-13T14:29:38,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178269 2024-11-13T14:29:38,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178269 2024-11-13T14:29:38,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178269) roll requested 2024-11-13T14:29:38,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178278 2024-11-13T14:29:38,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178278 2024-11-13T14:29:38,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178278) roll requested 2024-11-13T14:29:38,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178287 2024-11-13T14:29:38,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178287 2024-11-13T14:29:38,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178287) roll requested 2024-11-13T14:29:38,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178294 2024-11-13T14:29:38,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178294 2024-11-13T14:29:38,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178294) roll requested 2024-11-13T14:29:38,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178302 2024-11-13T14:29:38,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178302 2024-11-13T14:29:38,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178302) roll requested 2024-11-13T14:29:38,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178310 2024-11-13T14:29:38,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178310 2024-11-13T14:29:38,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178310) roll requested 2024-11-13T14:29:38,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178319 2024-11-13T14:29:38,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178319 2024-11-13T14:29:38,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178319) roll requested 2024-11-13T14:29:38,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178328 2024-11-13T14:29:38,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178328 2024-11-13T14:29:38,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178328) roll requested 2024-11-13T14:29:38,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178337 2024-11-13T14:29:38,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178337 2024-11-13T14:29:38,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178337) roll requested 2024-11-13T14:29:38,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178347 2024-11-13T14:29:38,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178347 2024-11-13T14:29:38,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178347) roll requested 2024-11-13T14:29:38,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178355 2024-11-13T14:29:38,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178355 2024-11-13T14:29:38,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178355) roll requested 2024-11-13T14:29:38,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178360 2024-11-13T14:29:38,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178360 2024-11-13T14:29:38,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178360) roll requested 2024-11-13T14:29:38,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178372 2024-11-13T14:29:38,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178372 2024-11-13T14:29:38,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178372) roll requested 2024-11-13T14:29:38,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178378 2024-11-13T14:29:38,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178378 2024-11-13T14:29:38,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178378) roll requested 2024-11-13T14:29:38,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178383 2024-11-13T14:29:38,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178383 2024-11-13T14:29:38,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178383) roll requested 2024-11-13T14:29:38,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178389 2024-11-13T14:29:38,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178389 2024-11-13T14:29:38,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178389) roll requested 2024-11-13T14:29:38,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178394 2024-11-13T14:29:38,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178394 2024-11-13T14:29:38,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178394) roll requested 2024-11-13T14:29:38,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178400 2024-11-13T14:29:38,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178400 2024-11-13T14:29:38,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178400) roll requested 2024-11-13T14:29:38,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178406 2024-11-13T14:29:38,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178406 2024-11-13T14:29:38,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178406) roll requested 2024-11-13T14:29:38,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178411 2024-11-13T14:29:38,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178411 2024-11-13T14:29:38,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178411) roll requested 2024-11-13T14:29:38,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178416 2024-11-13T14:29:38,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178416 2024-11-13T14:29:38,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178416) roll requested 2024-11-13T14:29:38,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178424 2024-11-13T14:29:38,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178424 2024-11-13T14:29:38,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178424) roll requested 2024-11-13T14:29:38,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178430 2024-11-13T14:29:38,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178430 2024-11-13T14:29:38,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178430) roll requested 2024-11-13T14:29:38,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178436 2024-11-13T14:29:38,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178436 2024-11-13T14:29:38,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178436) roll requested 2024-11-13T14:29:38,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178441 2024-11-13T14:29:38,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178441 2024-11-13T14:29:38,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178441) roll requested 2024-11-13T14:29:38,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178447 2024-11-13T14:29:38,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178447 2024-11-13T14:29:38,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178447) roll requested 2024-11-13T14:29:38,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178452 2024-11-13T14:29:38,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178452 2024-11-13T14:29:38,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178452) roll requested 2024-11-13T14:29:38,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178457 2024-11-13T14:29:38,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178457 2024-11-13T14:29:38,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178457) roll requested 2024-11-13T14:29:38,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178463 2024-11-13T14:29:38,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178463 2024-11-13T14:29:38,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178463) roll requested 2024-11-13T14:29:38,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178469 2024-11-13T14:29:38,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178469 2024-11-13T14:29:38,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178469) roll requested 2024-11-13T14:29:38,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178474 2024-11-13T14:29:38,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178474 2024-11-13T14:29:38,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178474) roll requested 2024-11-13T14:29:38,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178480 2024-11-13T14:29:38,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178480 2024-11-13T14:29:38,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178480) roll requested 2024-11-13T14:29:38,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178488 2024-11-13T14:29:38,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178488 2024-11-13T14:29:38,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178488) roll requested 2024-11-13T14:29:38,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178497 2024-11-13T14:29:38,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178497 2024-11-13T14:29:38,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178497) roll requested 2024-11-13T14:29:38,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178503 2024-11-13T14:29:38,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178503 2024-11-13T14:29:38,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178503) roll requested 2024-11-13T14:29:38,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178509 2024-11-13T14:29:38,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178509 2024-11-13T14:29:38,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178509) roll requested 2024-11-13T14:29:38,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178514 2024-11-13T14:29:38,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178514 2024-11-13T14:29:38,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178514) roll requested 2024-11-13T14:29:38,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178519 2024-11-13T14:29:38,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178519 2024-11-13T14:29:38,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178519) roll requested 2024-11-13T14:29:38,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178524 2024-11-13T14:29:38,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178524 2024-11-13T14:29:38,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178524) roll requested 2024-11-13T14:29:38,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178529 2024-11-13T14:29:38,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178529 2024-11-13T14:29:38,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178529) roll requested 2024-11-13T14:29:38,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178535 2024-11-13T14:29:38,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178535 2024-11-13T14:29:38,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178535) roll requested 2024-11-13T14:29:38,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178541 2024-11-13T14:29:38,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178541 2024-11-13T14:29:38,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178541) roll requested 2024-11-13T14:29:38,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178549 2024-11-13T14:29:38,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178549 2024-11-13T14:29:38,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178549) roll requested 2024-11-13T14:29:38,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178556 2024-11-13T14:29:38,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178556 2024-11-13T14:29:38,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178556) roll requested 2024-11-13T14:29:38,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178565 2024-11-13T14:29:38,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178565 2024-11-13T14:29:38,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178565) roll requested 2024-11-13T14:29:38,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178572 2024-11-13T14:29:38,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178572 2024-11-13T14:29:38,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178572) roll requested 2024-11-13T14:29:38,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178580 2024-11-13T14:29:38,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178580 2024-11-13T14:29:38,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178580) roll requested 2024-11-13T14:29:38,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178589 2024-11-13T14:29:38,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178589 2024-11-13T14:29:38,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178589) roll requested 2024-11-13T14:29:38,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178595 2024-11-13T14:29:38,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178595 2024-11-13T14:29:38,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178595) roll requested 2024-11-13T14:29:38,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178601 2024-11-13T14:29:38,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178601 2024-11-13T14:29:38,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178601) roll requested 2024-11-13T14:29:38,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178608 2024-11-13T14:29:38,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178608 2024-11-13T14:29:38,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178608) roll requested 2024-11-13T14:29:38,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178615 2024-11-13T14:29:38,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178615 2024-11-13T14:29:38,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178615) roll requested 2024-11-13T14:29:38,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178624 2024-11-13T14:29:38,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178624 2024-11-13T14:29:38,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178624) roll requested 2024-11-13T14:29:38,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178633 2024-11-13T14:29:38,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178633 2024-11-13T14:29:38,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178633) roll requested 2024-11-13T14:29:38,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178642 2024-11-13T14:29:38,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178642 2024-11-13T14:29:38,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178642) roll requested 2024-11-13T14:29:38,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178650 2024-11-13T14:29:38,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178650 2024-11-13T14:29:38,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178650) roll requested 2024-11-13T14:29:38,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178657 2024-11-13T14:29:38,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178657 2024-11-13T14:29:38,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178657) roll requested 2024-11-13T14:29:38,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178662 2024-11-13T14:29:38,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178662 2024-11-13T14:29:38,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178662) roll requested 2024-11-13T14:29:38,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178669 2024-11-13T14:29:38,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178669 2024-11-13T14:29:38,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178669) roll requested 2024-11-13T14:29:38,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178677 2024-11-13T14:29:38,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178677 2024-11-13T14:29:38,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178677) roll requested 2024-11-13T14:29:38,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178686 2024-11-13T14:29:38,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178686 2024-11-13T14:29:38,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178686) roll requested 2024-11-13T14:29:38,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178695 2024-11-13T14:29:38,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178695 2024-11-13T14:29:38,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178695) roll requested 2024-11-13T14:29:38,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178701 2024-11-13T14:29:38,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178701 2024-11-13T14:29:38,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178701) roll requested 2024-11-13T14:29:38,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178708 2024-11-13T14:29:38,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178708 2024-11-13T14:29:38,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178708) roll requested 2024-11-13T14:29:38,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178715 2024-11-13T14:29:38,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178715 2024-11-13T14:29:38,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178715) roll requested 2024-11-13T14:29:38,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178723 2024-11-13T14:29:38,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178723 2024-11-13T14:29:38,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178723) roll requested 2024-11-13T14:29:38,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178732 2024-11-13T14:29:38,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178732 2024-11-13T14:29:38,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178732) roll requested 2024-11-13T14:29:38,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178743 2024-11-13T14:29:38,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178743 2024-11-13T14:29:38,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178743) roll requested 2024-11-13T14:29:38,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178753 2024-11-13T14:29:38,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178753 2024-11-13T14:29:38,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178753) roll requested 2024-11-13T14:29:38,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178760 2024-11-13T14:29:38,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178760 2024-11-13T14:29:38,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178760) roll requested 2024-11-13T14:29:38,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178766 2024-11-13T14:29:38,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178766 2024-11-13T14:29:38,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178766) roll requested 2024-11-13T14:29:38,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178772 2024-11-13T14:29:38,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178772 2024-11-13T14:29:38,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178772) roll requested 2024-11-13T14:29:38,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178779 2024-11-13T14:29:38,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178779 2024-11-13T14:29:38,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178779) roll requested 2024-11-13T14:29:38,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178787 2024-11-13T14:29:38,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178787 2024-11-13T14:29:38,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178787) roll requested 2024-11-13T14:29:38,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178794 2024-11-13T14:29:38,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178794 2024-11-13T14:29:38,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178794) roll requested 2024-11-13T14:29:38,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178799 2024-11-13T14:29:38,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178799 2024-11-13T14:29:38,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178799) roll requested 2024-11-13T14:29:38,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178804 2024-11-13T14:29:38,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178804 2024-11-13T14:29:38,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178804) roll requested 2024-11-13T14:29:38,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178811 2024-11-13T14:29:38,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178811 2024-11-13T14:29:38,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178811) roll requested 2024-11-13T14:29:38,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178818 2024-11-13T14:29:38,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178818 2024-11-13T14:29:38,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178818) roll requested 2024-11-13T14:29:38,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178826 2024-11-13T14:29:38,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178826 2024-11-13T14:29:38,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178826) roll requested 2024-11-13T14:29:38,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178836 2024-11-13T14:29:38,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178836 2024-11-13T14:29:38,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178836) roll requested 2024-11-13T14:29:38,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178844 2024-11-13T14:29:38,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178844 2024-11-13T14:29:38,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178844) roll requested 2024-11-13T14:29:38,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178851 2024-11-13T14:29:38,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178851 2024-11-13T14:29:38,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178851) roll requested 2024-11-13T14:29:38,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178859 2024-11-13T14:29:38,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178859 2024-11-13T14:29:38,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178859) roll requested 2024-11-13T14:29:38,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178868 2024-11-13T14:29:38,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178868 2024-11-13T14:29:38,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178868) roll requested 2024-11-13T14:29:38,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178878 2024-11-13T14:29:38,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178878 2024-11-13T14:29:38,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178878) roll requested 2024-11-13T14:29:38,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178887 2024-11-13T14:29:38,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178887 2024-11-13T14:29:38,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178887) roll requested 2024-11-13T14:29:38,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178896 2024-11-13T14:29:38,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178896 2024-11-13T14:29:38,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178896) roll requested 2024-11-13T14:29:38,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178907 2024-11-13T14:29:38,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178907 2024-11-13T14:29:38,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178907) roll requested 2024-11-13T14:29:38,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178915 2024-11-13T14:29:38,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178915 2024-11-13T14:29:38,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178915) roll requested 2024-11-13T14:29:38,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178924 2024-11-13T14:29:38,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178924 2024-11-13T14:29:38,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178924) roll requested 2024-11-13T14:29:38,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178934 2024-11-13T14:29:38,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178934 2024-11-13T14:29:38,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178934) roll requested 2024-11-13T14:29:38,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178943 2024-11-13T14:29:38,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178943 2024-11-13T14:29:38,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178943) roll requested 2024-11-13T14:29:38,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178952 2024-11-13T14:29:38,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178952 2024-11-13T14:29:38,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178952) roll requested 2024-11-13T14:29:38,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178959 2024-11-13T14:29:38,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178959 2024-11-13T14:29:38,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178959) roll requested 2024-11-13T14:29:38,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178966 2024-11-13T14:29:38,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178966 2024-11-13T14:29:38,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178966) roll requested 2024-11-13T14:29:38,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178974 2024-11-13T14:29:38,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178974 2024-11-13T14:29:38,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178974) roll requested 2024-11-13T14:29:38,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178982 2024-11-13T14:29:38,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:38,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178982 2024-11-13T14:29:38,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:38,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:38,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178982) roll requested 2024-11-13T14:29:38,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508178991 2024-11-13T14:29:39,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178991 2024-11-13T14:29:39,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508178991) roll requested 2024-11-13T14:29:39,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179004 2024-11-13T14:29:39,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508178991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179004 2024-11-13T14:29:39,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179004) roll requested 2024-11-13T14:29:39,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179010 2024-11-13T14:29:39,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179010 2024-11-13T14:29:39,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179010) roll requested 2024-11-13T14:29:39,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179016 2024-11-13T14:29:39,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179016 2024-11-13T14:29:39,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179016) roll requested 2024-11-13T14:29:39,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179021 2024-11-13T14:29:39,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179021 2024-11-13T14:29:39,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179021) roll requested 2024-11-13T14:29:39,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179027 2024-11-13T14:29:39,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179027 2024-11-13T14:29:39,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179027) roll requested 2024-11-13T14:29:39,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179032 2024-11-13T14:29:39,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179032 2024-11-13T14:29:39,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179032) roll requested 2024-11-13T14:29:39,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179039 2024-11-13T14:29:39,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179039 2024-11-13T14:29:39,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179039) roll requested 2024-11-13T14:29:39,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179046 2024-11-13T14:29:39,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179046 2024-11-13T14:29:39,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179046) roll requested 2024-11-13T14:29:39,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179052 2024-11-13T14:29:39,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179052 2024-11-13T14:29:39,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179052) roll requested 2024-11-13T14:29:39,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179059 2024-11-13T14:29:39,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179059 2024-11-13T14:29:39,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179059) roll requested 2024-11-13T14:29:39,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179067 2024-11-13T14:29:39,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179067 2024-11-13T14:29:39,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179067) roll requested 2024-11-13T14:29:39,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179075 2024-11-13T14:29:39,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179075 2024-11-13T14:29:39,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179075) roll requested 2024-11-13T14:29:39,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179086 2024-11-13T14:29:39,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179086 2024-11-13T14:29:39,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179086) roll requested 2024-11-13T14:29:39,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179093 2024-11-13T14:29:39,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179093 2024-11-13T14:29:39,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179093) roll requested 2024-11-13T14:29:39,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179100 2024-11-13T14:29:39,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179100 2024-11-13T14:29:39,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179100) roll requested 2024-11-13T14:29:39,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179108 2024-11-13T14:29:39,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179108 2024-11-13T14:29:39,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179108) roll requested 2024-11-13T14:29:39,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179117 2024-11-13T14:29:39,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179117 2024-11-13T14:29:39,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179117) roll requested 2024-11-13T14:29:39,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179123 2024-11-13T14:29:39,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179123 2024-11-13T14:29:39,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179123) roll requested 2024-11-13T14:29:39,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179130 2024-11-13T14:29:39,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179130 2024-11-13T14:29:39,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179130) roll requested 2024-11-13T14:29:39,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179138 2024-11-13T14:29:39,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179138 2024-11-13T14:29:39,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179138) roll requested 2024-11-13T14:29:39,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179147 2024-11-13T14:29:39,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179147 2024-11-13T14:29:39,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179147) roll requested 2024-11-13T14:29:39,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179156 2024-11-13T14:29:39,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179156 2024-11-13T14:29:39,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179156) roll requested 2024-11-13T14:29:39,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179165 2024-11-13T14:29:39,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179165 2024-11-13T14:29:39,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179165) roll requested 2024-11-13T14:29:39,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179173 2024-11-13T14:29:39,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179173 2024-11-13T14:29:39,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179173) roll requested 2024-11-13T14:29:39,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179183 2024-11-13T14:29:39,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179183 2024-11-13T14:29:39,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179183) roll requested 2024-11-13T14:29:39,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179193 2024-11-13T14:29:39,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179193 2024-11-13T14:29:39,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179193) roll requested 2024-11-13T14:29:39,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179199 2024-11-13T14:29:39,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179199 2024-11-13T14:29:39,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179199) roll requested 2024-11-13T14:29:39,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179206 2024-11-13T14:29:39,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179206 2024-11-13T14:29:39,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179206) roll requested 2024-11-13T14:29:39,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179215 2024-11-13T14:29:39,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179215 2024-11-13T14:29:39,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179215) roll requested 2024-11-13T14:29:39,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179223 2024-11-13T14:29:39,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179223 2024-11-13T14:29:39,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179223) roll requested 2024-11-13T14:29:39,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179230 2024-11-13T14:29:39,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179230 2024-11-13T14:29:39,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179230) roll requested 2024-11-13T14:29:39,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179238 2024-11-13T14:29:39,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179238 2024-11-13T14:29:39,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179238) roll requested 2024-11-13T14:29:39,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179247 2024-11-13T14:29:39,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179247 2024-11-13T14:29:39,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179247) roll requested 2024-11-13T14:29:39,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179255 2024-11-13T14:29:39,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179255 2024-11-13T14:29:39,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179255) roll requested 2024-11-13T14:29:39,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179265 2024-11-13T14:29:39,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179265 2024-11-13T14:29:39,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179265) roll requested 2024-11-13T14:29:39,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179274 2024-11-13T14:29:39,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179274 2024-11-13T14:29:39,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179274) roll requested 2024-11-13T14:29:39,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179281 2024-11-13T14:29:39,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179281 2024-11-13T14:29:39,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179281) roll requested 2024-11-13T14:29:39,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179288 2024-11-13T14:29:39,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179288 2024-11-13T14:29:39,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179288) roll requested 2024-11-13T14:29:39,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179293 2024-11-13T14:29:39,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179293 2024-11-13T14:29:39,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179293) roll requested 2024-11-13T14:29:39,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179297 2024-11-13T14:29:39,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179297 2024-11-13T14:29:39,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179297) roll requested 2024-11-13T14:29:39,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179302 2024-11-13T14:29:39,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179302 2024-11-13T14:29:39,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179302) roll requested 2024-11-13T14:29:39,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179308 2024-11-13T14:29:39,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179308 2024-11-13T14:29:39,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179308) roll requested 2024-11-13T14:29:39,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179315 2024-11-13T14:29:39,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179315 2024-11-13T14:29:39,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179315) roll requested 2024-11-13T14:29:39,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179323 2024-11-13T14:29:39,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179323 2024-11-13T14:29:39,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179323) roll requested 2024-11-13T14:29:39,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179331 2024-11-13T14:29:39,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179331 2024-11-13T14:29:39,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179331) roll requested 2024-11-13T14:29:39,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179340 2024-11-13T14:29:39,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179340 2024-11-13T14:29:39,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179340) roll requested 2024-11-13T14:29:39,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179352 2024-11-13T14:29:39,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179352 2024-11-13T14:29:39,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179352) roll requested 2024-11-13T14:29:39,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179357 2024-11-13T14:29:39,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179357 2024-11-13T14:29:39,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179357) roll requested 2024-11-13T14:29:39,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179362 2024-11-13T14:29:39,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179362 2024-11-13T14:29:39,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179362) roll requested 2024-11-13T14:29:39,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179367 2024-11-13T14:29:39,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179367 2024-11-13T14:29:39,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179367) roll requested 2024-11-13T14:29:39,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179372 2024-11-13T14:29:39,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179372 2024-11-13T14:29:39,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179372) roll requested 2024-11-13T14:29:39,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179377 2024-11-13T14:29:39,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179377 2024-11-13T14:29:39,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179377) roll requested 2024-11-13T14:29:39,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179382 2024-11-13T14:29:39,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179382 2024-11-13T14:29:39,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179382) roll requested 2024-11-13T14:29:39,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179388 2024-11-13T14:29:39,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179388 2024-11-13T14:29:39,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179388) roll requested 2024-11-13T14:29:39,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179394 2024-11-13T14:29:39,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179394 2024-11-13T14:29:39,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179394) roll requested 2024-11-13T14:29:39,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179400 2024-11-13T14:29:39,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179400 2024-11-13T14:29:39,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179400) roll requested 2024-11-13T14:29:39,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179406 2024-11-13T14:29:39,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179406 2024-11-13T14:29:39,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179406) roll requested 2024-11-13T14:29:39,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179415 2024-11-13T14:29:39,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179415 2024-11-13T14:29:39,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179415) roll requested 2024-11-13T14:29:39,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179424 2024-11-13T14:29:39,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179424 2024-11-13T14:29:39,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179424) roll requested 2024-11-13T14:29:39,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179434 2024-11-13T14:29:39,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179434 2024-11-13T14:29:39,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179434) roll requested 2024-11-13T14:29:39,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179444 2024-11-13T14:29:39,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179444 2024-11-13T14:29:39,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179444) roll requested 2024-11-13T14:29:39,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179452 2024-11-13T14:29:39,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179452 2024-11-13T14:29:39,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179452) roll requested 2024-11-13T14:29:39,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179459 2024-11-13T14:29:39,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179459 2024-11-13T14:29:39,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179459) roll requested 2024-11-13T14:29:39,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179466 2024-11-13T14:29:39,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179466 2024-11-13T14:29:39,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179466) roll requested 2024-11-13T14:29:39,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179475 2024-11-13T14:29:39,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179475 2024-11-13T14:29:39,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179475) roll requested 2024-11-13T14:29:39,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179483 2024-11-13T14:29:39,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179483 2024-11-13T14:29:39,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179483) roll requested 2024-11-13T14:29:39,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179490 2024-11-13T14:29:39,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179490 2024-11-13T14:29:39,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179490) roll requested 2024-11-13T14:29:39,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179499 2024-11-13T14:29:39,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179499 2024-11-13T14:29:39,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179499) roll requested 2024-11-13T14:29:39,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179507 2024-11-13T14:29:39,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179507 2024-11-13T14:29:39,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179507) roll requested 2024-11-13T14:29:39,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179516 2024-11-13T14:29:39,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179516 2024-11-13T14:29:39,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179516) roll requested 2024-11-13T14:29:39,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179525 2024-11-13T14:29:39,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179525 2024-11-13T14:29:39,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179525) roll requested 2024-11-13T14:29:39,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179533 2024-11-13T14:29:39,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179533 2024-11-13T14:29:39,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179533) roll requested 2024-11-13T14:29:39,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179541 2024-11-13T14:29:39,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179541 2024-11-13T14:29:39,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179541) roll requested 2024-11-13T14:29:39,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179549 2024-11-13T14:29:39,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179549 2024-11-13T14:29:39,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179549) roll requested 2024-11-13T14:29:39,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179555 2024-11-13T14:29:39,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179555 2024-11-13T14:29:39,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179555) roll requested 2024-11-13T14:29:39,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179563 2024-11-13T14:29:39,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179563 2024-11-13T14:29:39,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179563) roll requested 2024-11-13T14:29:39,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179571 2024-11-13T14:29:39,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179571 2024-11-13T14:29:39,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179571) roll requested 2024-11-13T14:29:39,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179580 2024-11-13T14:29:39,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179580 2024-11-13T14:29:39,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179580) roll requested 2024-11-13T14:29:39,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179588 2024-11-13T14:29:39,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179588 2024-11-13T14:29:39,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179588) roll requested 2024-11-13T14:29:39,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179598 2024-11-13T14:29:39,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179598 2024-11-13T14:29:39,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179598) roll requested 2024-11-13T14:29:39,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179604 2024-11-13T14:29:39,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179604 2024-11-13T14:29:39,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179604) roll requested 2024-11-13T14:29:39,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179612 2024-11-13T14:29:39,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179612 2024-11-13T14:29:39,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179612) roll requested 2024-11-13T14:29:39,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179621 2024-11-13T14:29:39,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179621 2024-11-13T14:29:39,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179621) roll requested 2024-11-13T14:29:39,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179626 2024-11-13T14:29:39,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179626 2024-11-13T14:29:39,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179626) roll requested 2024-11-13T14:29:39,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179631 2024-11-13T14:29:39,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179631 2024-11-13T14:29:39,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179631) roll requested 2024-11-13T14:29:39,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179637 2024-11-13T14:29:39,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179637 2024-11-13T14:29:39,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179637) roll requested 2024-11-13T14:29:39,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179644 2024-11-13T14:29:39,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179644 2024-11-13T14:29:39,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179644) roll requested 2024-11-13T14:29:39,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179649 2024-11-13T14:29:39,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179649 2024-11-13T14:29:39,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179649) roll requested 2024-11-13T14:29:39,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179654 2024-11-13T14:29:39,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179654 2024-11-13T14:29:39,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179654) roll requested 2024-11-13T14:29:39,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179660 2024-11-13T14:29:39,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179660 2024-11-13T14:29:39,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179660) roll requested 2024-11-13T14:29:39,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179665 2024-11-13T14:29:39,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179665 2024-11-13T14:29:39,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179665) roll requested 2024-11-13T14:29:39,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179671 2024-11-13T14:29:39,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179671 2024-11-13T14:29:39,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179671) roll requested 2024-11-13T14:29:39,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179676 2024-11-13T14:29:39,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179676 2024-11-13T14:29:39,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179676) roll requested 2024-11-13T14:29:39,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179681 2024-11-13T14:29:39,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179681 2024-11-13T14:29:39,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179681) roll requested 2024-11-13T14:29:39,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179687 2024-11-13T14:29:39,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179687 2024-11-13T14:29:39,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179687) roll requested 2024-11-13T14:29:39,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179692 2024-11-13T14:29:39,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179692 2024-11-13T14:29:39,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179692) roll requested 2024-11-13T14:29:39,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179698 2024-11-13T14:29:39,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179698 2024-11-13T14:29:39,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179698) roll requested 2024-11-13T14:29:39,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179704 2024-11-13T14:29:39,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179704 2024-11-13T14:29:39,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179704) roll requested 2024-11-13T14:29:39,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179709 2024-11-13T14:29:39,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179709 2024-11-13T14:29:39,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179709) roll requested 2024-11-13T14:29:39,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179715 2024-11-13T14:29:39,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179715 2024-11-13T14:29:39,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179715) roll requested 2024-11-13T14:29:39,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179721 2024-11-13T14:29:39,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179721 2024-11-13T14:29:39,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179721) roll requested 2024-11-13T14:29:39,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179726 2024-11-13T14:29:39,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179726 2024-11-13T14:29:39,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179726) roll requested 2024-11-13T14:29:39,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179732 2024-11-13T14:29:39,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179732 2024-11-13T14:29:39,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179732) roll requested 2024-11-13T14:29:39,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179737 2024-11-13T14:29:39,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179737 2024-11-13T14:29:39,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179737) roll requested 2024-11-13T14:29:39,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179742 2024-11-13T14:29:39,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179742 2024-11-13T14:29:39,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179742) roll requested 2024-11-13T14:29:39,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179747 2024-11-13T14:29:39,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179747 2024-11-13T14:29:39,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179747) roll requested 2024-11-13T14:29:39,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179752 2024-11-13T14:29:39,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179752 2024-11-13T14:29:39,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179752) roll requested 2024-11-13T14:29:39,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179758 2024-11-13T14:29:39,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179758 2024-11-13T14:29:39,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179758) roll requested 2024-11-13T14:29:39,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179763 2024-11-13T14:29:39,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179763 2024-11-13T14:29:39,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179763) roll requested 2024-11-13T14:29:39,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179768 2024-11-13T14:29:39,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179768 2024-11-13T14:29:39,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179768) roll requested 2024-11-13T14:29:39,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179773 2024-11-13T14:29:39,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179773 2024-11-13T14:29:39,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179773) roll requested 2024-11-13T14:29:39,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179780 2024-11-13T14:29:39,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179780 2024-11-13T14:29:39,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179780) roll requested 2024-11-13T14:29:39,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179786 2024-11-13T14:29:39,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179786 2024-11-13T14:29:39,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179786) roll requested 2024-11-13T14:29:39,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179792 2024-11-13T14:29:39,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179792 2024-11-13T14:29:39,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179792) roll requested 2024-11-13T14:29:39,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179797 2024-11-13T14:29:39,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179797 2024-11-13T14:29:39,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179797) roll requested 2024-11-13T14:29:39,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179803 2024-11-13T14:29:39,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179803 2024-11-13T14:29:39,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179803) roll requested 2024-11-13T14:29:39,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179810 2024-11-13T14:29:39,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179810 2024-11-13T14:29:39,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179810) roll requested 2024-11-13T14:29:39,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179815 2024-11-13T14:29:39,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179815 2024-11-13T14:29:39,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179815) roll requested 2024-11-13T14:29:39,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179821 2024-11-13T14:29:39,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179821 2024-11-13T14:29:39,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179821) roll requested 2024-11-13T14:29:39,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179826 2024-11-13T14:29:39,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179826 2024-11-13T14:29:39,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179826) roll requested 2024-11-13T14:29:39,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179831 2024-11-13T14:29:39,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179831 2024-11-13T14:29:39,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179831) roll requested 2024-11-13T14:29:39,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179836 2024-11-13T14:29:39,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179836 2024-11-13T14:29:39,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179836) roll requested 2024-11-13T14:29:39,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179841 2024-11-13T14:29:39,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179841 2024-11-13T14:29:39,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179841) roll requested 2024-11-13T14:29:39,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179847 2024-11-13T14:29:39,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179847 2024-11-13T14:29:39,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179847) roll requested 2024-11-13T14:29:39,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179852 2024-11-13T14:29:39,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179852 2024-11-13T14:29:39,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179852) roll requested 2024-11-13T14:29:39,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179857 2024-11-13T14:29:39,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179857 2024-11-13T14:29:39,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179857) roll requested 2024-11-13T14:29:39,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179862 2024-11-13T14:29:39,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179862 2024-11-13T14:29:39,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179862) roll requested 2024-11-13T14:29:39,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179868 2024-11-13T14:29:39,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179868 2024-11-13T14:29:39,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179868) roll requested 2024-11-13T14:29:39,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179874 2024-11-13T14:29:39,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179874 2024-11-13T14:29:39,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179874) roll requested 2024-11-13T14:29:39,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179879 2024-11-13T14:29:39,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179879 2024-11-13T14:29:39,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179879) roll requested 2024-11-13T14:29:39,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179884 2024-11-13T14:29:39,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179884 2024-11-13T14:29:39,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179884) roll requested 2024-11-13T14:29:39,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179890 2024-11-13T14:29:39,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179890 2024-11-13T14:29:39,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179890) roll requested 2024-11-13T14:29:39,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179896 2024-11-13T14:29:39,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179896 2024-11-13T14:29:39,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179896) roll requested 2024-11-13T14:29:39,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179903 2024-11-13T14:29:39,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179903 2024-11-13T14:29:39,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179903) roll requested 2024-11-13T14:29:39,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179910 2024-11-13T14:29:39,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179910 2024-11-13T14:29:39,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179910) roll requested 2024-11-13T14:29:39,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179918 2024-11-13T14:29:39,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179918 2024-11-13T14:29:39,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179918) roll requested 2024-11-13T14:29:39,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179926 2024-11-13T14:29:39,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179926 2024-11-13T14:29:39,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179926) roll requested 2024-11-13T14:29:39,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179934 2024-11-13T14:29:39,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179934 2024-11-13T14:29:39,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179934) roll requested 2024-11-13T14:29:39,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179943 2024-11-13T14:29:39,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179943 2024-11-13T14:29:39,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179943) roll requested 2024-11-13T14:29:39,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179950 2024-11-13T14:29:39,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179950 2024-11-13T14:29:39,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179950) roll requested 2024-11-13T14:29:39,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179958 2024-11-13T14:29:39,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179958 2024-11-13T14:29:39,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179958) roll requested 2024-11-13T14:29:39,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179965 2024-11-13T14:29:39,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179965 2024-11-13T14:29:39,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179965) roll requested 2024-11-13T14:29:39,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179973 2024-11-13T14:29:39,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179973 2024-11-13T14:29:39,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179973) roll requested 2024-11-13T14:29:39,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179981 2024-11-13T14:29:39,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179981 2024-11-13T14:29:39,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179981) roll requested 2024-11-13T14:29:39,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179989 2024-11-13T14:29:39,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:39,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179989 2024-11-13T14:29:39,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:39,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:39,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179989) roll requested 2024-11-13T14:29:39,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508179998 2024-11-13T14:29:40,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179998 2024-11-13T14:29:40,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508179998) roll requested 2024-11-13T14:29:40,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180006 2024-11-13T14:29:40,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508179998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180006 2024-11-13T14:29:40,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180006) roll requested 2024-11-13T14:29:40,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180014 2024-11-13T14:29:40,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180014 2024-11-13T14:29:40,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180014) roll requested 2024-11-13T14:29:40,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180024 2024-11-13T14:29:40,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180024 2024-11-13T14:29:40,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180024) roll requested 2024-11-13T14:29:40,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180034 2024-11-13T14:29:40,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180034 2024-11-13T14:29:40,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180034) roll requested 2024-11-13T14:29:40,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180044 2024-11-13T14:29:40,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180044 2024-11-13T14:29:40,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180044) roll requested 2024-11-13T14:29:40,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180051 2024-11-13T14:29:40,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180051 2024-11-13T14:29:40,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180051) roll requested 2024-11-13T14:29:40,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180060 2024-11-13T14:29:40,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180060 2024-11-13T14:29:40,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180060) roll requested 2024-11-13T14:29:40,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180070 2024-11-13T14:29:40,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180070 2024-11-13T14:29:40,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180070) roll requested 2024-11-13T14:29:40,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180079 2024-11-13T14:29:40,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180079 2024-11-13T14:29:40,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180079) roll requested 2024-11-13T14:29:40,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180085 2024-11-13T14:29:40,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180085 2024-11-13T14:29:40,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180085) roll requested 2024-11-13T14:29:40,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180091 2024-11-13T14:29:40,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180091 2024-11-13T14:29:40,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180091) roll requested 2024-11-13T14:29:40,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180099 2024-11-13T14:29:40,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180099 2024-11-13T14:29:40,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180099) roll requested 2024-11-13T14:29:40,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180107 2024-11-13T14:29:40,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180107 2024-11-13T14:29:40,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180107) roll requested 2024-11-13T14:29:40,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180115 2024-11-13T14:29:40,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180115 2024-11-13T14:29:40,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180115) roll requested 2024-11-13T14:29:40,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180125 2024-11-13T14:29:40,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180125 2024-11-13T14:29:40,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180125) roll requested 2024-11-13T14:29:40,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180136 2024-11-13T14:29:40,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180136 2024-11-13T14:29:40,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180136) roll requested 2024-11-13T14:29:40,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180142 2024-11-13T14:29:40,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180142 2024-11-13T14:29:40,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180142) roll requested 2024-11-13T14:29:40,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180147 2024-11-13T14:29:40,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180147 2024-11-13T14:29:40,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180147) roll requested 2024-11-13T14:29:40,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180153 2024-11-13T14:29:40,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180153 2024-11-13T14:29:40,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180153) roll requested 2024-11-13T14:29:40,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180160 2024-11-13T14:29:40,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180160 2024-11-13T14:29:40,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180160) roll requested 2024-11-13T14:29:40,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180170 2024-11-13T14:29:40,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180170 2024-11-13T14:29:40,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180170) roll requested 2024-11-13T14:29:40,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180179 2024-11-13T14:29:40,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180179 2024-11-13T14:29:40,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180179) roll requested 2024-11-13T14:29:40,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180187 2024-11-13T14:29:40,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180187 2024-11-13T14:29:40,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180187) roll requested 2024-11-13T14:29:40,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180194 2024-11-13T14:29:40,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180194 2024-11-13T14:29:40,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180194) roll requested 2024-11-13T14:29:40,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180201 2024-11-13T14:29:40,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180201 2024-11-13T14:29:40,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180201) roll requested 2024-11-13T14:29:40,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180208 2024-11-13T14:29:40,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180208 2024-11-13T14:29:40,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180208) roll requested 2024-11-13T14:29:40,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180216 2024-11-13T14:29:40,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180216 2024-11-13T14:29:40,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180216) roll requested 2024-11-13T14:29:40,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180223 2024-11-13T14:29:40,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180223 2024-11-13T14:29:40,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180223) roll requested 2024-11-13T14:29:40,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180234 2024-11-13T14:29:40,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180234 2024-11-13T14:29:40,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180234) roll requested 2024-11-13T14:29:40,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180242 2024-11-13T14:29:40,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180242 2024-11-13T14:29:40,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180242) roll requested 2024-11-13T14:29:40,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180249 2024-11-13T14:29:40,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180249 2024-11-13T14:29:40,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180249) roll requested 2024-11-13T14:29:40,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180257 2024-11-13T14:29:40,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180257 2024-11-13T14:29:40,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180257) roll requested 2024-11-13T14:29:40,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180264 2024-11-13T14:29:40,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180264 2024-11-13T14:29:40,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180264) roll requested 2024-11-13T14:29:40,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180272 2024-11-13T14:29:40,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180272 2024-11-13T14:29:40,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180272) roll requested 2024-11-13T14:29:40,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180280 2024-11-13T14:29:40,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180280 2024-11-13T14:29:40,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180280) roll requested 2024-11-13T14:29:40,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180288 2024-11-13T14:29:40,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180288 2024-11-13T14:29:40,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180288) roll requested 2024-11-13T14:29:40,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180300 2024-11-13T14:29:40,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180300 2024-11-13T14:29:40,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180300) roll requested 2024-11-13T14:29:40,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180305 2024-11-13T14:29:40,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180305 2024-11-13T14:29:40,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180305) roll requested 2024-11-13T14:29:40,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180310 2024-11-13T14:29:40,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180310 2024-11-13T14:29:40,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180310) roll requested 2024-11-13T14:29:40,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180315 2024-11-13T14:29:40,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180315 2024-11-13T14:29:40,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180315) roll requested 2024-11-13T14:29:40,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180320 2024-11-13T14:29:40,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180320 2024-11-13T14:29:40,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180320) roll requested 2024-11-13T14:29:40,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180326 2024-11-13T14:29:40,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180326 2024-11-13T14:29:40,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180326) roll requested 2024-11-13T14:29:40,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180334 2024-11-13T14:29:40,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180334 2024-11-13T14:29:40,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180334) roll requested 2024-11-13T14:29:40,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180344 2024-11-13T14:29:40,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180344 2024-11-13T14:29:40,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180344) roll requested 2024-11-13T14:29:40,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180354 2024-11-13T14:29:40,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180354 2024-11-13T14:29:40,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180354) roll requested 2024-11-13T14:29:40,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180363 2024-11-13T14:29:40,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180363 2024-11-13T14:29:40,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180363) roll requested 2024-11-13T14:29:40,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180371 2024-11-13T14:29:40,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180371 2024-11-13T14:29:40,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180371) roll requested 2024-11-13T14:29:40,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180378 2024-11-13T14:29:40,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180378 2024-11-13T14:29:40,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180378) roll requested 2024-11-13T14:29:40,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180386 2024-11-13T14:29:40,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180386 2024-11-13T14:29:40,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180386) roll requested 2024-11-13T14:29:40,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180394 2024-11-13T14:29:40,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180394 2024-11-13T14:29:40,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180394) roll requested 2024-11-13T14:29:40,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180402 2024-11-13T14:29:40,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180402 2024-11-13T14:29:40,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180402) roll requested 2024-11-13T14:29:40,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180410 2024-11-13T14:29:40,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180410 2024-11-13T14:29:40,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180410) roll requested 2024-11-13T14:29:40,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180418 2024-11-13T14:29:40,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180418 2024-11-13T14:29:40,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180418) roll requested 2024-11-13T14:29:40,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180426 2024-11-13T14:29:40,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180426 2024-11-13T14:29:40,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180426) roll requested 2024-11-13T14:29:40,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180434 2024-11-13T14:29:40,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180434 2024-11-13T14:29:40,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180434) roll requested 2024-11-13T14:29:40,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180442 2024-11-13T14:29:40,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180442 2024-11-13T14:29:40,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180442) roll requested 2024-11-13T14:29:40,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180449 2024-11-13T14:29:40,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180449 2024-11-13T14:29:40,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180449) roll requested 2024-11-13T14:29:40,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180455 2024-11-13T14:29:40,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180455 2024-11-13T14:29:40,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180455) roll requested 2024-11-13T14:29:40,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180461 2024-11-13T14:29:40,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180461 2024-11-13T14:29:40,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180461) roll requested 2024-11-13T14:29:40,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180467 2024-11-13T14:29:40,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180467 2024-11-13T14:29:40,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180467) roll requested 2024-11-13T14:29:40,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180475 2024-11-13T14:29:40,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180475 2024-11-13T14:29:40,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180475) roll requested 2024-11-13T14:29:40,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180484 2024-11-13T14:29:40,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180484 2024-11-13T14:29:40,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180484) roll requested 2024-11-13T14:29:40,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180493 2024-11-13T14:29:40,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180493 2024-11-13T14:29:40,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180493) roll requested 2024-11-13T14:29:40,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180500 2024-11-13T14:29:40,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180500 2024-11-13T14:29:40,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180500) roll requested 2024-11-13T14:29:40,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180508 2024-11-13T14:29:40,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180508 2024-11-13T14:29:40,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180508) roll requested 2024-11-13T14:29:40,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180516 2024-11-13T14:29:40,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180516 2024-11-13T14:29:40,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180516) roll requested 2024-11-13T14:29:40,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180523 2024-11-13T14:29:40,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180523 2024-11-13T14:29:40,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180523) roll requested 2024-11-13T14:29:40,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180531 2024-11-13T14:29:40,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180531 2024-11-13T14:29:40,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180531) roll requested 2024-11-13T14:29:40,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180540 2024-11-13T14:29:40,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180540 2024-11-13T14:29:40,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180540) roll requested 2024-11-13T14:29:40,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180550 2024-11-13T14:29:40,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180550 2024-11-13T14:29:40,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180550) roll requested 2024-11-13T14:29:40,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180558 2024-11-13T14:29:40,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180558 2024-11-13T14:29:40,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180558) roll requested 2024-11-13T14:29:40,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180564 2024-11-13T14:29:40,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180564 2024-11-13T14:29:40,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180564) roll requested 2024-11-13T14:29:40,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180569 2024-11-13T14:29:40,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180569 2024-11-13T14:29:40,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180569) roll requested 2024-11-13T14:29:40,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180575 2024-11-13T14:29:40,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180575 2024-11-13T14:29:40,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180575) roll requested 2024-11-13T14:29:40,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180580 2024-11-13T14:29:40,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180580 2024-11-13T14:29:40,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180580) roll requested 2024-11-13T14:29:40,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180585 2024-11-13T14:29:40,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180585 2024-11-13T14:29:40,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180585) roll requested 2024-11-13T14:29:40,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180590 2024-11-13T14:29:40,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180590 2024-11-13T14:29:40,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180590) roll requested 2024-11-13T14:29:40,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180597 2024-11-13T14:29:40,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180597 2024-11-13T14:29:40,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180597) roll requested 2024-11-13T14:29:40,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180604 2024-11-13T14:29:40,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180604 2024-11-13T14:29:40,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180604) roll requested 2024-11-13T14:29:40,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180612 2024-11-13T14:29:40,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180612 2024-11-13T14:29:40,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180612) roll requested 2024-11-13T14:29:40,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180621 2024-11-13T14:29:40,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180621 2024-11-13T14:29:40,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180621) roll requested 2024-11-13T14:29:40,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180629 2024-11-13T14:29:40,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180629 2024-11-13T14:29:40,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180629) roll requested 2024-11-13T14:29:40,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180637 2024-11-13T14:29:40,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180637 2024-11-13T14:29:40,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180637) roll requested 2024-11-13T14:29:40,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180645 2024-11-13T14:29:40,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180645 2024-11-13T14:29:40,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180645) roll requested 2024-11-13T14:29:40,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180651 2024-11-13T14:29:40,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180651 2024-11-13T14:29:40,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180651) roll requested 2024-11-13T14:29:40,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180660 2024-11-13T14:29:40,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180660 2024-11-13T14:29:40,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180660) roll requested 2024-11-13T14:29:40,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180668 2024-11-13T14:29:40,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180668 2024-11-13T14:29:40,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180668) roll requested 2024-11-13T14:29:40,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180976 2024-11-13T14:29:40,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180976 2024-11-13T14:29:40,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180976) roll requested 2024-11-13T14:29:40,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180986 2024-11-13T14:29:40,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180986 2024-11-13T14:29:40,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180986) roll requested 2024-11-13T14:29:40,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180992 2024-11-13T14:29:40,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:40,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180992 2024-11-13T14:29:40,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:40,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:40,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180992) roll requested 2024-11-13T14:29:40,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508180998 2024-11-13T14:29:41,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180998 2024-11-13T14:29:41,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508180998) roll requested 2024-11-13T14:29:41,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181006 2024-11-13T14:29:41,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508180998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181006 2024-11-13T14:29:41,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181006) roll requested 2024-11-13T14:29:41,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181014 2024-11-13T14:29:41,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181014 2024-11-13T14:29:41,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181014) roll requested 2024-11-13T14:29:41,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181022 2024-11-13T14:29:41,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181022 2024-11-13T14:29:41,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181022) roll requested 2024-11-13T14:29:41,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181031 2024-11-13T14:29:41,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181031 2024-11-13T14:29:41,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181031) roll requested 2024-11-13T14:29:41,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181039 2024-11-13T14:29:41,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181039 2024-11-13T14:29:41,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181039) roll requested 2024-11-13T14:29:41,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181046 2024-11-13T14:29:41,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181046 2024-11-13T14:29:41,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181046) roll requested 2024-11-13T14:29:41,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181055 2024-11-13T14:29:41,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181055 2024-11-13T14:29:41,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181055) roll requested 2024-11-13T14:29:41,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181064 2024-11-13T14:29:41,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181064 2024-11-13T14:29:41,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181064) roll requested 2024-11-13T14:29:41,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181073 2024-11-13T14:29:41,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181073 2024-11-13T14:29:41,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181073) roll requested 2024-11-13T14:29:41,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181083 2024-11-13T14:29:41,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181083 2024-11-13T14:29:41,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181083) roll requested 2024-11-13T14:29:41,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181091 2024-11-13T14:29:41,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181091 2024-11-13T14:29:41,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181091) roll requested 2024-11-13T14:29:41,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181099 2024-11-13T14:29:41,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181099 2024-11-13T14:29:41,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181099) roll requested 2024-11-13T14:29:41,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181107 2024-11-13T14:29:41,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181107 2024-11-13T14:29:41,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181107) roll requested 2024-11-13T14:29:41,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181117 2024-11-13T14:29:41,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181117 2024-11-13T14:29:41,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181117) roll requested 2024-11-13T14:29:41,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181125 2024-11-13T14:29:41,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181125 2024-11-13T14:29:41,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181125) roll requested 2024-11-13T14:29:41,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181133 2024-11-13T14:29:41,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181133 2024-11-13T14:29:41,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181133) roll requested 2024-11-13T14:29:41,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181142 2024-11-13T14:29:41,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181142 2024-11-13T14:29:41,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181142) roll requested 2024-11-13T14:29:41,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181151 2024-11-13T14:29:41,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181151 2024-11-13T14:29:41,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181151) roll requested 2024-11-13T14:29:41,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181159 2024-11-13T14:29:41,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181159 2024-11-13T14:29:41,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181159) roll requested 2024-11-13T14:29:41,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181167 2024-11-13T14:29:41,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181167 2024-11-13T14:29:41,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181167) roll requested 2024-11-13T14:29:41,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181176 2024-11-13T14:29:41,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181176 2024-11-13T14:29:41,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181176) roll requested 2024-11-13T14:29:41,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181183 2024-11-13T14:29:41,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181183 2024-11-13T14:29:41,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181183) roll requested 2024-11-13T14:29:41,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181190 2024-11-13T14:29:41,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181190 2024-11-13T14:29:41,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181190) roll requested 2024-11-13T14:29:41,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181196 2024-11-13T14:29:41,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181196 2024-11-13T14:29:41,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181196) roll requested 2024-11-13T14:29:41,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181204 2024-11-13T14:29:41,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181204 2024-11-13T14:29:41,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181204) roll requested 2024-11-13T14:29:41,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181213 2024-11-13T14:29:41,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181213 2024-11-13T14:29:41,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181213) roll requested 2024-11-13T14:29:41,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181222 2024-11-13T14:29:41,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181222 2024-11-13T14:29:41,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181222) roll requested 2024-11-13T14:29:41,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181229 2024-11-13T14:29:41,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181229 2024-11-13T14:29:41,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181229) roll requested 2024-11-13T14:29:41,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181236 2024-11-13T14:29:41,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181236 2024-11-13T14:29:41,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181236) roll requested 2024-11-13T14:29:41,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181242 2024-11-13T14:29:41,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181242 2024-11-13T14:29:41,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181242) roll requested 2024-11-13T14:29:41,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181249 2024-11-13T14:29:41,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181249 2024-11-13T14:29:41,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181249) roll requested 2024-11-13T14:29:41,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181255 2024-11-13T14:29:41,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181255 2024-11-13T14:29:41,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181255) roll requested 2024-11-13T14:29:41,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181263 2024-11-13T14:29:41,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181263 2024-11-13T14:29:41,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181263) roll requested 2024-11-13T14:29:41,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181271 2024-11-13T14:29:41,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181271 2024-11-13T14:29:41,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181271) roll requested 2024-11-13T14:29:41,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181279 2024-11-13T14:29:41,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181279 2024-11-13T14:29:41,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181279) roll requested 2024-11-13T14:29:41,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181286 2024-11-13T14:29:41,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181286 2024-11-13T14:29:41,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181286) roll requested 2024-11-13T14:29:41,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181293 2024-11-13T14:29:41,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181293 2024-11-13T14:29:41,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181293) roll requested 2024-11-13T14:29:41,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181302 2024-11-13T14:29:41,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181302 2024-11-13T14:29:41,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181302) roll requested 2024-11-13T14:29:41,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181310 2024-11-13T14:29:41,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181310 2024-11-13T14:29:41,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181310) roll requested 2024-11-13T14:29:41,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181316 2024-11-13T14:29:41,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181316 2024-11-13T14:29:41,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181316) roll requested 2024-11-13T14:29:41,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181323 2024-11-13T14:29:41,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181323 2024-11-13T14:29:41,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181323) roll requested 2024-11-13T14:29:41,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181331 2024-11-13T14:29:41,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181331 2024-11-13T14:29:41,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181331) roll requested 2024-11-13T14:29:41,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181339 2024-11-13T14:29:41,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181339 2024-11-13T14:29:41,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181339) roll requested 2024-11-13T14:29:41,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181348 2024-11-13T14:29:41,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181348 2024-11-13T14:29:41,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181348) roll requested 2024-11-13T14:29:41,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181356 2024-11-13T14:29:41,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181356 2024-11-13T14:29:41,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181356) roll requested 2024-11-13T14:29:41,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181365 2024-11-13T14:29:41,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181365 2024-11-13T14:29:41,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181365) roll requested 2024-11-13T14:29:41,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181375 2024-11-13T14:29:41,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181375 2024-11-13T14:29:41,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181375) roll requested 2024-11-13T14:29:41,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181384 2024-11-13T14:29:41,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181384 2024-11-13T14:29:41,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181384) roll requested 2024-11-13T14:29:41,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181394 2024-11-13T14:29:41,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181394 2024-11-13T14:29:41,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181394) roll requested 2024-11-13T14:29:41,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181404 2024-11-13T14:29:41,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181404 2024-11-13T14:29:41,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181404) roll requested 2024-11-13T14:29:41,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181412 2024-11-13T14:29:41,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181412 2024-11-13T14:29:41,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181412) roll requested 2024-11-13T14:29:41,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181420 2024-11-13T14:29:41,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181420 2024-11-13T14:29:41,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181420) roll requested 2024-11-13T14:29:41,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181427 2024-11-13T14:29:41,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181427 2024-11-13T14:29:41,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181427) roll requested 2024-11-13T14:29:41,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181437 2024-11-13T14:29:41,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181437 2024-11-13T14:29:41,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181437) roll requested 2024-11-13T14:29:41,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181446 2024-11-13T14:29:41,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181446 2024-11-13T14:29:41,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181446) roll requested 2024-11-13T14:29:41,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181457 2024-11-13T14:29:41,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181457 2024-11-13T14:29:41,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181457) roll requested 2024-11-13T14:29:41,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181463 2024-11-13T14:29:41,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181463 2024-11-13T14:29:41,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181463) roll requested 2024-11-13T14:29:41,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181468 2024-11-13T14:29:41,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181468 2024-11-13T14:29:41,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181468) roll requested 2024-11-13T14:29:41,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181475 2024-11-13T14:29:41,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181475 2024-11-13T14:29:41,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181475) roll requested 2024-11-13T14:29:41,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181483 2024-11-13T14:29:41,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181483 2024-11-13T14:29:41,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181483) roll requested 2024-11-13T14:29:41,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181491 2024-11-13T14:29:41,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181491 2024-11-13T14:29:41,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181491) roll requested 2024-11-13T14:29:41,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181500 2024-11-13T14:29:41,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181500 2024-11-13T14:29:41,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181500) roll requested 2024-11-13T14:29:41,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181508 2024-11-13T14:29:41,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181508 2024-11-13T14:29:41,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181508) roll requested 2024-11-13T14:29:41,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181513 2024-11-13T14:29:41,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181513 2024-11-13T14:29:41,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181513) roll requested 2024-11-13T14:29:41,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181519 2024-11-13T14:29:41,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181519 2024-11-13T14:29:41,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181519) roll requested 2024-11-13T14:29:41,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181524 2024-11-13T14:29:41,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181524 2024-11-13T14:29:41,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181524) roll requested 2024-11-13T14:29:41,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181528 2024-11-13T14:29:41,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181528 2024-11-13T14:29:41,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181528) roll requested 2024-11-13T14:29:41,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181534 2024-11-13T14:29:41,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181534 2024-11-13T14:29:41,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181534) roll requested 2024-11-13T14:29:41,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181539 2024-11-13T14:29:41,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181539 2024-11-13T14:29:41,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181539) roll requested 2024-11-13T14:29:41,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181545 2024-11-13T14:29:41,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181545 2024-11-13T14:29:41,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181545) roll requested 2024-11-13T14:29:41,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181552 2024-11-13T14:29:41,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181552 2024-11-13T14:29:41,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181552) roll requested 2024-11-13T14:29:41,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181558 2024-11-13T14:29:41,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181558 2024-11-13T14:29:41,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181558) roll requested 2024-11-13T14:29:41,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181564 2024-11-13T14:29:41,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181564 2024-11-13T14:29:41,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181564) roll requested 2024-11-13T14:29:41,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181569 2024-11-13T14:29:41,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181569 2024-11-13T14:29:41,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181569) roll requested 2024-11-13T14:29:41,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181575 2024-11-13T14:29:41,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181575 2024-11-13T14:29:41,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181575) roll requested 2024-11-13T14:29:41,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181582 2024-11-13T14:29:41,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181582 2024-11-13T14:29:41,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181582) roll requested 2024-11-13T14:29:41,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181588 2024-11-13T14:29:41,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181588 2024-11-13T14:29:41,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181588) roll requested 2024-11-13T14:29:41,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181596 2024-11-13T14:29:41,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181596 2024-11-13T14:29:41,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181596) roll requested 2024-11-13T14:29:41,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181604 2024-11-13T14:29:41,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181604 2024-11-13T14:29:41,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181604) roll requested 2024-11-13T14:29:41,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181612 2024-11-13T14:29:41,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181612 2024-11-13T14:29:41,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181612) roll requested 2024-11-13T14:29:41,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181619 2024-11-13T14:29:41,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181619 2024-11-13T14:29:41,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181619) roll requested 2024-11-13T14:29:41,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181627 2024-11-13T14:29:41,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181627 2024-11-13T14:29:41,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181627) roll requested 2024-11-13T14:29:41,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181634 2024-11-13T14:29:41,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181634 2024-11-13T14:29:41,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181634) roll requested 2024-11-13T14:29:41,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181640 2024-11-13T14:29:41,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181640 2024-11-13T14:29:41,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181640) roll requested 2024-11-13T14:29:41,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181647 2024-11-13T14:29:41,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181647 2024-11-13T14:29:41,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181647) roll requested 2024-11-13T14:29:41,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181654 2024-11-13T14:29:41,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181654 2024-11-13T14:29:41,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181654) roll requested 2024-11-13T14:29:41,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181661 2024-11-13T14:29:41,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181661 2024-11-13T14:29:41,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181661) roll requested 2024-11-13T14:29:41,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181669 2024-11-13T14:29:41,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181669 2024-11-13T14:29:41,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181669) roll requested 2024-11-13T14:29:41,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181679 2024-11-13T14:29:41,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181679 2024-11-13T14:29:41,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181679) roll requested 2024-11-13T14:29:41,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181687 2024-11-13T14:29:41,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181687 2024-11-13T14:29:41,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181687) roll requested 2024-11-13T14:29:41,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181695 2024-11-13T14:29:41,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181695 2024-11-13T14:29:41,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181695) roll requested 2024-11-13T14:29:41,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181702 2024-11-13T14:29:41,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181702 2024-11-13T14:29:41,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181702) roll requested 2024-11-13T14:29:41,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181709 2024-11-13T14:29:41,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181709 2024-11-13T14:29:41,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181709) roll requested 2024-11-13T14:29:41,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181717 2024-11-13T14:29:41,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181717 2024-11-13T14:29:41,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181717) roll requested 2024-11-13T14:29:41,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181724 2024-11-13T14:29:41,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181724 2024-11-13T14:29:41,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181724) roll requested 2024-11-13T14:29:41,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181733 2024-11-13T14:29:41,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181733 2024-11-13T14:29:41,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181733) roll requested 2024-11-13T14:29:41,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181741 2024-11-13T14:29:41,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181741 2024-11-13T14:29:41,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181741) roll requested 2024-11-13T14:29:41,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181747 2024-11-13T14:29:41,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181747 2024-11-13T14:29:41,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181747) roll requested 2024-11-13T14:29:41,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181754 2024-11-13T14:29:41,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181754 2024-11-13T14:29:41,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181754) roll requested 2024-11-13T14:29:41,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181761 2024-11-13T14:29:41,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181761 2024-11-13T14:29:41,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181761) roll requested 2024-11-13T14:29:41,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181768 2024-11-13T14:29:41,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181768 2024-11-13T14:29:41,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181768) roll requested 2024-11-13T14:29:41,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181775 2024-11-13T14:29:41,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181775 2024-11-13T14:29:41,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181775) roll requested 2024-11-13T14:29:41,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181783 2024-11-13T14:29:41,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181783 2024-11-13T14:29:41,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181783) roll requested 2024-11-13T14:29:41,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181792 2024-11-13T14:29:41,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181792 2024-11-13T14:29:41,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181792) roll requested 2024-11-13T14:29:41,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181799 2024-11-13T14:29:41,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181799 2024-11-13T14:29:41,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181799) roll requested 2024-11-13T14:29:41,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181806 2024-11-13T14:29:41,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181806 2024-11-13T14:29:41,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181806) roll requested 2024-11-13T14:29:41,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181814 2024-11-13T14:29:41,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181814 2024-11-13T14:29:41,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181814) roll requested 2024-11-13T14:29:41,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181823 2024-11-13T14:29:41,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181823 2024-11-13T14:29:41,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181823) roll requested 2024-11-13T14:29:41,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181831 2024-11-13T14:29:41,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181831 2024-11-13T14:29:41,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181831) roll requested 2024-11-13T14:29:41,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181839 2024-11-13T14:29:41,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181839 2024-11-13T14:29:41,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181839) roll requested 2024-11-13T14:29:41,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181848 2024-11-13T14:29:41,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181848 2024-11-13T14:29:41,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181848) roll requested 2024-11-13T14:29:41,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181858 2024-11-13T14:29:41,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181858 2024-11-13T14:29:41,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181858) roll requested 2024-11-13T14:29:41,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181868 2024-11-13T14:29:41,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181868 2024-11-13T14:29:41,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181868) roll requested 2024-11-13T14:29:41,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181877 2024-11-13T14:29:41,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181877 2024-11-13T14:29:41,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181877) roll requested 2024-11-13T14:29:41,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181887 2024-11-13T14:29:41,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181887 2024-11-13T14:29:41,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181887) roll requested 2024-11-13T14:29:41,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181895 2024-11-13T14:29:41,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181895 2024-11-13T14:29:41,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181895) roll requested 2024-11-13T14:29:41,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181901 2024-11-13T14:29:41,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181901 2024-11-13T14:29:41,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181901) roll requested 2024-11-13T14:29:41,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181907 2024-11-13T14:29:41,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181907 2024-11-13T14:29:41,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181907) roll requested 2024-11-13T14:29:41,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181914 2024-11-13T14:29:41,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181914 2024-11-13T14:29:41,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181914) roll requested 2024-11-13T14:29:41,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181921 2024-11-13T14:29:41,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181921 2024-11-13T14:29:41,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181921) roll requested 2024-11-13T14:29:41,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181929 2024-11-13T14:29:41,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181929 2024-11-13T14:29:41,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181929) roll requested 2024-11-13T14:29:41,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181937 2024-11-13T14:29:41,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181937 2024-11-13T14:29:41,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181937) roll requested 2024-11-13T14:29:41,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181946 2024-11-13T14:29:41,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181946 2024-11-13T14:29:41,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181946) roll requested 2024-11-13T14:29:41,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181952 2024-11-13T14:29:41,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181952 2024-11-13T14:29:41,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181952) roll requested 2024-11-13T14:29:41,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181961 2024-11-13T14:29:41,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181961 2024-11-13T14:29:41,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181961) roll requested 2024-11-13T14:29:41,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181969 2024-11-13T14:29:41,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181969 2024-11-13T14:29:41,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181969) roll requested 2024-11-13T14:29:41,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181977 2024-11-13T14:29:41,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181977 2024-11-13T14:29:41,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181977) roll requested 2024-11-13T14:29:41,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181985 2024-11-13T14:29:41,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181985 2024-11-13T14:29:41,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:41,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:41,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181985) roll requested 2024-11-13T14:29:41,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508181993 2024-11-13T14:29:41,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:41,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181993 2024-11-13T14:29:42,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508181993) roll requested 2024-11-13T14:29:42,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182000 2024-11-13T14:29:42,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508181993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182000 2024-11-13T14:29:42,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182000) roll requested 2024-11-13T14:29:42,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182007 2024-11-13T14:29:42,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182007 2024-11-13T14:29:42,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182007) roll requested 2024-11-13T14:29:42,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182015 2024-11-13T14:29:42,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182015 2024-11-13T14:29:42,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182015) roll requested 2024-11-13T14:29:42,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182024 2024-11-13T14:29:42,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182024 2024-11-13T14:29:42,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182024) roll requested 2024-11-13T14:29:42,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182032 2024-11-13T14:29:42,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182032 2024-11-13T14:29:42,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182032) roll requested 2024-11-13T14:29:42,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182041 2024-11-13T14:29:42,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182041 2024-11-13T14:29:42,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182041) roll requested 2024-11-13T14:29:42,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182050 2024-11-13T14:29:42,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182050 2024-11-13T14:29:42,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182050) roll requested 2024-11-13T14:29:42,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182057 2024-11-13T14:29:42,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182057 2024-11-13T14:29:42,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182057) roll requested 2024-11-13T14:29:42,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182068 2024-11-13T14:29:42,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182068 2024-11-13T14:29:42,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182068) roll requested 2024-11-13T14:29:42,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182077 2024-11-13T14:29:42,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182077 2024-11-13T14:29:42,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182077) roll requested 2024-11-13T14:29:42,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182083 2024-11-13T14:29:42,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182083 2024-11-13T14:29:42,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182083) roll requested 2024-11-13T14:29:42,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182089 2024-11-13T14:29:42,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182089 2024-11-13T14:29:42,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182089) roll requested 2024-11-13T14:29:42,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182094 2024-11-13T14:29:42,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182094 2024-11-13T14:29:42,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182094) roll requested 2024-11-13T14:29:42,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182102 2024-11-13T14:29:42,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182102 2024-11-13T14:29:42,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182102) roll requested 2024-11-13T14:29:42,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182110 2024-11-13T14:29:42,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182110 2024-11-13T14:29:42,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182110) roll requested 2024-11-13T14:29:42,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182120 2024-11-13T14:29:42,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182120 2024-11-13T14:29:42,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182120) roll requested 2024-11-13T14:29:42,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182127 2024-11-13T14:29:42,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182127 2024-11-13T14:29:42,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182127) roll requested 2024-11-13T14:29:42,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182135 2024-11-13T14:29:42,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182135 2024-11-13T14:29:42,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182135) roll requested 2024-11-13T14:29:42,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182145 2024-11-13T14:29:42,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182145 2024-11-13T14:29:42,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182145) roll requested 2024-11-13T14:29:42,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182151 2024-11-13T14:29:42,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182151 2024-11-13T14:29:42,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182151) roll requested 2024-11-13T14:29:42,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182157 2024-11-13T14:29:42,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182157 2024-11-13T14:29:42,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182157) roll requested 2024-11-13T14:29:42,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182163 2024-11-13T14:29:42,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182163 2024-11-13T14:29:42,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182163) roll requested 2024-11-13T14:29:42,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182169 2024-11-13T14:29:42,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182169 2024-11-13T14:29:42,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182169) roll requested 2024-11-13T14:29:42,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182180 2024-11-13T14:29:42,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182180 2024-11-13T14:29:42,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182180) roll requested 2024-11-13T14:29:42,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182188 2024-11-13T14:29:42,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182188 2024-11-13T14:29:42,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182188) roll requested 2024-11-13T14:29:42,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182194 2024-11-13T14:29:42,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182194 2024-11-13T14:29:42,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182194) roll requested 2024-11-13T14:29:42,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182199 2024-11-13T14:29:42,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182199 2024-11-13T14:29:42,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182199) roll requested 2024-11-13T14:29:42,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182204 2024-11-13T14:29:42,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182204 2024-11-13T14:29:42,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182204) roll requested 2024-11-13T14:29:42,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182210 2024-11-13T14:29:42,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182210 2024-11-13T14:29:42,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182210) roll requested 2024-11-13T14:29:42,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182220 2024-11-13T14:29:42,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182220 2024-11-13T14:29:42,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182220) roll requested 2024-11-13T14:29:42,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182227 2024-11-13T14:29:42,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182227 2024-11-13T14:29:42,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182227) roll requested 2024-11-13T14:29:42,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182231 2024-11-13T14:29:42,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182231 2024-11-13T14:29:42,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182231) roll requested 2024-11-13T14:29:42,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182236 2024-11-13T14:29:42,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182236 2024-11-13T14:29:42,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182236) roll requested 2024-11-13T14:29:42,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182242 2024-11-13T14:29:42,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182242 2024-11-13T14:29:42,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182242) roll requested 2024-11-13T14:29:42,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182249 2024-11-13T14:29:42,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182249 2024-11-13T14:29:42,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182249) roll requested 2024-11-13T14:29:42,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182256 2024-11-13T14:29:42,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182256 2024-11-13T14:29:42,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182256) roll requested 2024-11-13T14:29:42,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182262 2024-11-13T14:29:42,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182262 2024-11-13T14:29:42,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:42,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182262) roll requested 2024-11-13T14:29:42,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182273 2024-11-13T14:29:42,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182273 2024-11-13T14:29:42,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182273) roll requested 2024-11-13T14:29:42,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182280 2024-11-13T14:29:42,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182280 2024-11-13T14:29:42,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182280) roll requested 2024-11-13T14:29:42,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182286 2024-11-13T14:29:42,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182286 2024-11-13T14:29:42,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182286) roll requested 2024-11-13T14:29:42,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182297 2024-11-13T14:29:42,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182297 2024-11-13T14:29:42,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182297) roll requested 2024-11-13T14:29:42,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182303 2024-11-13T14:29:42,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182303 2024-11-13T14:29:42,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182303) roll requested 2024-11-13T14:29:42,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182310 2024-11-13T14:29:42,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182310 2024-11-13T14:29:42,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182310) roll requested 2024-11-13T14:29:42,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182316 2024-11-13T14:29:42,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182316 2024-11-13T14:29:42,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182316) roll requested 2024-11-13T14:29:42,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182322 2024-11-13T14:29:42,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182322 2024-11-13T14:29:42,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182322) roll requested 2024-11-13T14:29:42,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182330 2024-11-13T14:29:42,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182330 2024-11-13T14:29:42,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182330) roll requested 2024-11-13T14:29:42,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182340 2024-11-13T14:29:42,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182340 2024-11-13T14:29:42,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182340) roll requested 2024-11-13T14:29:42,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182349 2024-11-13T14:29:42,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182349 2024-11-13T14:29:42,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182349) roll requested 2024-11-13T14:29:42,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182364 2024-11-13T14:29:42,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182364 2024-11-13T14:29:42,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182364) roll requested 2024-11-13T14:29:42,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182370 2024-11-13T14:29:42,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182370 2024-11-13T14:29:42,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182370) roll requested 2024-11-13T14:29:42,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182376 2024-11-13T14:29:42,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182376 2024-11-13T14:29:42,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182376) roll requested 2024-11-13T14:29:42,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182381 2024-11-13T14:29:42,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182381 2024-11-13T14:29:42,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182381) roll requested 2024-11-13T14:29:42,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182387 2024-11-13T14:29:42,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182387 2024-11-13T14:29:42,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182387) roll requested 2024-11-13T14:29:42,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182393 2024-11-13T14:29:42,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182393 2024-11-13T14:29:42,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182393) roll requested 2024-11-13T14:29:42,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182400 2024-11-13T14:29:42,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182400 2024-11-13T14:29:42,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182400) roll requested 2024-11-13T14:29:42,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182408 2024-11-13T14:29:42,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182408 2024-11-13T14:29:42,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182408) roll requested 2024-11-13T14:29:42,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182416 2024-11-13T14:29:42,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182416 2024-11-13T14:29:42,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182416) roll requested 2024-11-13T14:29:42,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182424 2024-11-13T14:29:42,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182424 2024-11-13T14:29:42,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182424) roll requested 2024-11-13T14:29:42,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182432 2024-11-13T14:29:42,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182432 2024-11-13T14:29:42,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182432) roll requested 2024-11-13T14:29:42,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182440 2024-11-13T14:29:42,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182440 2024-11-13T14:29:42,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182440) roll requested 2024-11-13T14:29:42,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182447 2024-11-13T14:29:42,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182447 2024-11-13T14:29:42,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182447) roll requested 2024-11-13T14:29:42,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182457 2024-11-13T14:29:42,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182457 2024-11-13T14:29:42,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182457) roll requested 2024-11-13T14:29:42,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182465 2024-11-13T14:29:42,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182465 2024-11-13T14:29:42,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182465) roll requested 2024-11-13T14:29:42,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182475 2024-11-13T14:29:42,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182475 2024-11-13T14:29:42,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182475) roll requested 2024-11-13T14:29:42,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182486 2024-11-13T14:29:42,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182486 2024-11-13T14:29:42,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182486) roll requested 2024-11-13T14:29:42,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182495 2024-11-13T14:29:42,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182495 2024-11-13T14:29:42,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182495) roll requested 2024-11-13T14:29:42,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182505 2024-11-13T14:29:42,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182505 2024-11-13T14:29:42,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182505) roll requested 2024-11-13T14:29:42,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182514 2024-11-13T14:29:42,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182514 2024-11-13T14:29:42,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182514) roll requested 2024-11-13T14:29:42,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182520 2024-11-13T14:29:42,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182520 2024-11-13T14:29:42,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182520) roll requested 2024-11-13T14:29:42,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182525 2024-11-13T14:29:42,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182525 2024-11-13T14:29:42,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182525) roll requested 2024-11-13T14:29:42,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182531 2024-11-13T14:29:42,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182531 2024-11-13T14:29:42,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182531) roll requested 2024-11-13T14:29:42,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182538 2024-11-13T14:29:42,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182538 2024-11-13T14:29:42,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182538) roll requested 2024-11-13T14:29:42,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182544 2024-11-13T14:29:42,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182544 2024-11-13T14:29:42,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182544) roll requested 2024-11-13T14:29:42,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182550 2024-11-13T14:29:42,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182550 2024-11-13T14:29:42,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182550) roll requested 2024-11-13T14:29:42,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182556 2024-11-13T14:29:42,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182556 2024-11-13T14:29:42,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182556) roll requested 2024-11-13T14:29:42,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182562 2024-11-13T14:29:42,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182562 2024-11-13T14:29:42,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182562) roll requested 2024-11-13T14:29:42,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182567 2024-11-13T14:29:42,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182567 2024-11-13T14:29:42,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182567) roll requested 2024-11-13T14:29:42,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182574 2024-11-13T14:29:42,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182574 2024-11-13T14:29:42,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182574) roll requested 2024-11-13T14:29:42,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182581 2024-11-13T14:29:42,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182581 2024-11-13T14:29:42,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182581) roll requested 2024-11-13T14:29:42,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182588 2024-11-13T14:29:42,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182588 2024-11-13T14:29:42,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182588) roll requested 2024-11-13T14:29:42,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182597 2024-11-13T14:29:42,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182597 2024-11-13T14:29:42,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182597) roll requested 2024-11-13T14:29:42,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182607 2024-11-13T14:29:42,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182607 2024-11-13T14:29:42,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182607) roll requested 2024-11-13T14:29:42,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182617 2024-11-13T14:29:42,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182617 2024-11-13T14:29:42,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182617) roll requested 2024-11-13T14:29:42,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182627 2024-11-13T14:29:42,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182627 2024-11-13T14:29:42,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182627) roll requested 2024-11-13T14:29:42,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182637 2024-11-13T14:29:42,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182637 2024-11-13T14:29:42,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182637) roll requested 2024-11-13T14:29:42,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182646 2024-11-13T14:29:42,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182646 2024-11-13T14:29:42,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182646) roll requested 2024-11-13T14:29:42,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182653 2024-11-13T14:29:42,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182653 2024-11-13T14:29:42,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182653) roll requested 2024-11-13T14:29:42,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182662 2024-11-13T14:29:42,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182662 2024-11-13T14:29:42,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182662) roll requested 2024-11-13T14:29:42,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182670 2024-11-13T14:29:42,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182670 2024-11-13T14:29:42,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182670) roll requested 2024-11-13T14:29:42,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182677 2024-11-13T14:29:42,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182677 2024-11-13T14:29:42,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182677) roll requested 2024-11-13T14:29:42,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182685 2024-11-13T14:29:42,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182685 2024-11-13T14:29:42,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182685) roll requested 2024-11-13T14:29:42,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182691 2024-11-13T14:29:42,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182691 2024-11-13T14:29:42,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182691) roll requested 2024-11-13T14:29:42,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182698 2024-11-13T14:29:42,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182698 2024-11-13T14:29:42,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182698) roll requested 2024-11-13T14:29:42,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182708 2024-11-13T14:29:42,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182708 2024-11-13T14:29:42,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182708) roll requested 2024-11-13T14:29:42,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182716 2024-11-13T14:29:42,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182716 2024-11-13T14:29:42,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182716) roll requested 2024-11-13T14:29:42,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182725 2024-11-13T14:29:42,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182725 2024-11-13T14:29:42,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182725) roll requested 2024-11-13T14:29:42,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182735 2024-11-13T14:29:42,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182735 2024-11-13T14:29:42,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182735) roll requested 2024-11-13T14:29:42,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182741 2024-11-13T14:29:42,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182741 2024-11-13T14:29:42,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182741) roll requested 2024-11-13T14:29:42,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182751 2024-11-13T14:29:42,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182751 2024-11-13T14:29:42,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182751) roll requested 2024-11-13T14:29:42,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182756 2024-11-13T14:29:42,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182756 2024-11-13T14:29:42,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182756) roll requested 2024-11-13T14:29:42,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182761 2024-11-13T14:29:42,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182761 2024-11-13T14:29:42,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182761) roll requested 2024-11-13T14:29:42,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182773 2024-11-13T14:29:42,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182773 2024-11-13T14:29:42,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182773) roll requested 2024-11-13T14:29:42,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182779 2024-11-13T14:29:42,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182779 2024-11-13T14:29:42,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182779) roll requested 2024-11-13T14:29:42,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182784 2024-11-13T14:29:42,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182784 2024-11-13T14:29:42,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182784) roll requested 2024-11-13T14:29:42,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182790 2024-11-13T14:29:42,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182790 2024-11-13T14:29:42,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182790) roll requested 2024-11-13T14:29:42,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182797 2024-11-13T14:29:42,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182797 2024-11-13T14:29:42,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182797) roll requested 2024-11-13T14:29:42,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182802 2024-11-13T14:29:42,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182802 2024-11-13T14:29:42,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182802) roll requested 2024-11-13T14:29:42,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182807 2024-11-13T14:29:42,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182807 2024-11-13T14:29:42,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182807) roll requested 2024-11-13T14:29:42,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182813 2024-11-13T14:29:42,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182813 2024-11-13T14:29:42,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182813) roll requested 2024-11-13T14:29:42,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182819 2024-11-13T14:29:42,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182819 2024-11-13T14:29:42,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,826 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,826 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182819) roll requested 2024-11-13T14:29:42,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182826 2024-11-13T14:29:42,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182826 2024-11-13T14:29:42,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182826) roll requested 2024-11-13T14:29:42,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182833 2024-11-13T14:29:42,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182833 2024-11-13T14:29:42,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182833) roll requested 2024-11-13T14:29:42,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182839 2024-11-13T14:29:42,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182839 2024-11-13T14:29:42,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182839) roll requested 2024-11-13T14:29:42,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182845 2024-11-13T14:29:42,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182845 2024-11-13T14:29:42,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182845) roll requested 2024-11-13T14:29:42,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182852 2024-11-13T14:29:42,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182852 2024-11-13T14:29:42,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182852) roll requested 2024-11-13T14:29:42,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182860 2024-11-13T14:29:42,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182860 2024-11-13T14:29:42,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182860) roll requested 2024-11-13T14:29:42,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182869 2024-11-13T14:29:42,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182869 2024-11-13T14:29:42,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182869) roll requested 2024-11-13T14:29:42,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182875 2024-11-13T14:29:42,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182875 2024-11-13T14:29:42,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182875) roll requested 2024-11-13T14:29:42,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182880 2024-11-13T14:29:42,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182880 2024-11-13T14:29:42,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182880) roll requested 2024-11-13T14:29:42,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182885 2024-11-13T14:29:42,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182885 2024-11-13T14:29:42,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182885) roll requested 2024-11-13T14:29:42,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182892 2024-11-13T14:29:42,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182892 2024-11-13T14:29:42,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182892) roll requested 2024-11-13T14:29:42,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182901 2024-11-13T14:29:42,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182901 2024-11-13T14:29:42,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182901) roll requested 2024-11-13T14:29:42,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182908 2024-11-13T14:29:42,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182908 2024-11-13T14:29:42,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182908) roll requested 2024-11-13T14:29:42,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182915 2024-11-13T14:29:42,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182915 2024-11-13T14:29:42,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182915) roll requested 2024-11-13T14:29:42,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182921 2024-11-13T14:29:42,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182921 2024-11-13T14:29:42,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182921) roll requested 2024-11-13T14:29:42,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182928 2024-11-13T14:29:42,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182928 2024-11-13T14:29:42,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182928) roll requested 2024-11-13T14:29:42,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182934 2024-11-13T14:29:42,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182934 2024-11-13T14:29:42,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182934) roll requested 2024-11-13T14:29:42,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182939 2024-11-13T14:29:42,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182939 2024-11-13T14:29:42,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182939) roll requested 2024-11-13T14:29:42,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182944 2024-11-13T14:29:42,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182944 2024-11-13T14:29:42,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182944) roll requested 2024-11-13T14:29:42,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182950 2024-11-13T14:29:42,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182950 2024-11-13T14:29:42,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182950) roll requested 2024-11-13T14:29:42,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182955 2024-11-13T14:29:42,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182955 2024-11-13T14:29:42,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182955) roll requested 2024-11-13T14:29:42,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182960 2024-11-13T14:29:42,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182960 2024-11-13T14:29:42,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182960) roll requested 2024-11-13T14:29:42,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182966 2024-11-13T14:29:42,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182966 2024-11-13T14:29:42,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182966) roll requested 2024-11-13T14:29:42,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182971 2024-11-13T14:29:42,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182971 2024-11-13T14:29:42,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182971) roll requested 2024-11-13T14:29:42,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182977 2024-11-13T14:29:42,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182977 2024-11-13T14:29:42,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182977) roll requested 2024-11-13T14:29:42,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182984 2024-11-13T14:29:42,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182984 2024-11-13T14:29:42,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:42,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:42,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:42,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:42,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182984) roll requested 2024-11-13T14:29:42,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508182991 2024-11-13T14:29:42,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:42,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182991 2024-11-13T14:29:43,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508182991) roll requested 2024-11-13T14:29:43,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183001 2024-11-13T14:29:43,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508182991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183001 2024-11-13T14:29:43,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183001) roll requested 2024-11-13T14:29:43,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183010 2024-11-13T14:29:43,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183010 2024-11-13T14:29:43,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183010) roll requested 2024-11-13T14:29:43,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183019 2024-11-13T14:29:43,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183019 2024-11-13T14:29:43,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183019) roll requested 2024-11-13T14:29:43,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183029 2024-11-13T14:29:43,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183029 2024-11-13T14:29:43,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183029) roll requested 2024-11-13T14:29:43,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183038 2024-11-13T14:29:43,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183038 2024-11-13T14:29:43,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183038) roll requested 2024-11-13T14:29:43,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183048 2024-11-13T14:29:43,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183048 2024-11-13T14:29:43,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183048) roll requested 2024-11-13T14:29:43,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183057 2024-11-13T14:29:43,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183057 2024-11-13T14:29:43,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183057) roll requested 2024-11-13T14:29:43,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183068 2024-11-13T14:29:43,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183068 2024-11-13T14:29:43,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183068) roll requested 2024-11-13T14:29:43,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183074 2024-11-13T14:29:43,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183074 2024-11-13T14:29:43,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183074) roll requested 2024-11-13T14:29:43,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183079 2024-11-13T14:29:43,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183079 2024-11-13T14:29:43,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183079) roll requested 2024-11-13T14:29:43,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183085 2024-11-13T14:29:43,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183085 2024-11-13T14:29:43,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183085) roll requested 2024-11-13T14:29:43,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183090 2024-11-13T14:29:43,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183090 2024-11-13T14:29:43,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183090) roll requested 2024-11-13T14:29:43,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183096 2024-11-13T14:29:43,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183096 2024-11-13T14:29:43,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183096) roll requested 2024-11-13T14:29:43,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183102 2024-11-13T14:29:43,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183102 2024-11-13T14:29:43,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183102) roll requested 2024-11-13T14:29:43,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183108 2024-11-13T14:29:43,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183108 2024-11-13T14:29:43,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183108) roll requested 2024-11-13T14:29:43,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183113 2024-11-13T14:29:43,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183113 2024-11-13T14:29:43,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183113) roll requested 2024-11-13T14:29:43,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183119 2024-11-13T14:29:43,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183119 2024-11-13T14:29:43,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183119) roll requested 2024-11-13T14:29:43,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183125 2024-11-13T14:29:43,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183125 2024-11-13T14:29:43,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183125) roll requested 2024-11-13T14:29:43,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183132 2024-11-13T14:29:43,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183132 2024-11-13T14:29:43,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183132) roll requested 2024-11-13T14:29:43,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183139 2024-11-13T14:29:43,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183139 2024-11-13T14:29:43,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183139) roll requested 2024-11-13T14:29:43,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183147 2024-11-13T14:29:43,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183147 2024-11-13T14:29:43,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183147) roll requested 2024-11-13T14:29:43,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183154 2024-11-13T14:29:43,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183154 2024-11-13T14:29:43,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183154) roll requested 2024-11-13T14:29:43,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183159 2024-11-13T14:29:43,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183159 2024-11-13T14:29:43,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183159) roll requested 2024-11-13T14:29:43,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183165 2024-11-13T14:29:43,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183165 2024-11-13T14:29:43,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183165) roll requested 2024-11-13T14:29:43,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183172 2024-11-13T14:29:43,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183172 2024-11-13T14:29:43,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183172) roll requested 2024-11-13T14:29:43,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183181 2024-11-13T14:29:43,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183181 2024-11-13T14:29:43,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183181) roll requested 2024-11-13T14:29:43,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183186 2024-11-13T14:29:43,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183186 2024-11-13T14:29:43,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183186) roll requested 2024-11-13T14:29:43,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183199 2024-11-13T14:29:43,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183199 2024-11-13T14:29:43,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183199) roll requested 2024-11-13T14:29:43,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183205 2024-11-13T14:29:43,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183205 2024-11-13T14:29:43,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183205) roll requested 2024-11-13T14:29:43,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183213 2024-11-13T14:29:43,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183213 2024-11-13T14:29:43,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183213) roll requested 2024-11-13T14:29:43,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183219 2024-11-13T14:29:43,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183219 2024-11-13T14:29:43,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183219) roll requested 2024-11-13T14:29:43,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183226 2024-11-13T14:29:43,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183226 2024-11-13T14:29:43,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183226) roll requested 2024-11-13T14:29:43,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183233 2024-11-13T14:29:43,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183233 2024-11-13T14:29:43,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183233) roll requested 2024-11-13T14:29:43,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183241 2024-11-13T14:29:43,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183241 2024-11-13T14:29:43,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183241) roll requested 2024-11-13T14:29:43,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183248 2024-11-13T14:29:43,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183248 2024-11-13T14:29:43,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183248) roll requested 2024-11-13T14:29:43,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183254 2024-11-13T14:29:43,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183254 2024-11-13T14:29:43,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183254) roll requested 2024-11-13T14:29:43,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183261 2024-11-13T14:29:43,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183261 2024-11-13T14:29:43,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183261) roll requested 2024-11-13T14:29:43,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183269 2024-11-13T14:29:43,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183269 2024-11-13T14:29:43,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183269) roll requested 2024-11-13T14:29:43,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183285 2024-11-13T14:29:43,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183285 2024-11-13T14:29:43,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183285) roll requested 2024-11-13T14:29:43,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183293 2024-11-13T14:29:43,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183293 2024-11-13T14:29:43,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183293) roll requested 2024-11-13T14:29:43,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183299 2024-11-13T14:29:43,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183299 2024-11-13T14:29:43,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183299) roll requested 2024-11-13T14:29:43,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183304 2024-11-13T14:29:43,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183304 2024-11-13T14:29:43,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183304) roll requested 2024-11-13T14:29:43,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183311 2024-11-13T14:29:43,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183311 2024-11-13T14:29:43,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183311) roll requested 2024-11-13T14:29:43,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183317 2024-11-13T14:29:43,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183317 2024-11-13T14:29:43,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183317) roll requested 2024-11-13T14:29:43,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183323 2024-11-13T14:29:43,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183323 2024-11-13T14:29:43,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183323) roll requested 2024-11-13T14:29:43,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183329 2024-11-13T14:29:43,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183329 2024-11-13T14:29:43,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183329) roll requested 2024-11-13T14:29:43,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183335 2024-11-13T14:29:43,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183335 2024-11-13T14:29:43,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183335) roll requested 2024-11-13T14:29:43,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183341 2024-11-13T14:29:43,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183341 2024-11-13T14:29:43,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183341) roll requested 2024-11-13T14:29:43,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183347 2024-11-13T14:29:43,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183347 2024-11-13T14:29:43,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183347) roll requested 2024-11-13T14:29:43,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183352 2024-11-13T14:29:43,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183352 2024-11-13T14:29:43,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183352) roll requested 2024-11-13T14:29:43,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183357 2024-11-13T14:29:43,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183357 2024-11-13T14:29:43,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183357) roll requested 2024-11-13T14:29:43,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183363 2024-11-13T14:29:43,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183363 2024-11-13T14:29:43,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183363) roll requested 2024-11-13T14:29:43,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183369 2024-11-13T14:29:43,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183369 2024-11-13T14:29:43,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183369) roll requested 2024-11-13T14:29:43,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183374 2024-11-13T14:29:43,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183374 2024-11-13T14:29:43,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183374) roll requested 2024-11-13T14:29:43,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183380 2024-11-13T14:29:43,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183380 2024-11-13T14:29:43,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183380) roll requested 2024-11-13T14:29:43,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183385 2024-11-13T14:29:43,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183385 2024-11-13T14:29:43,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183385) roll requested 2024-11-13T14:29:43,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183391 2024-11-13T14:29:43,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183391 2024-11-13T14:29:43,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183391) roll requested 2024-11-13T14:29:43,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183397 2024-11-13T14:29:43,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183397 2024-11-13T14:29:43,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183397) roll requested 2024-11-13T14:29:43,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183403 2024-11-13T14:29:43,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183403 2024-11-13T14:29:43,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183403) roll requested 2024-11-13T14:29:43,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183409 2024-11-13T14:29:43,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183409 2024-11-13T14:29:43,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183409) roll requested 2024-11-13T14:29:43,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183415 2024-11-13T14:29:43,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183415 2024-11-13T14:29:43,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183415) roll requested 2024-11-13T14:29:43,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183422 2024-11-13T14:29:43,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183422 2024-11-13T14:29:43,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183422) roll requested 2024-11-13T14:29:43,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183428 2024-11-13T14:29:43,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183428 2024-11-13T14:29:43,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183428) roll requested 2024-11-13T14:29:43,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183434 2024-11-13T14:29:43,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183434 2024-11-13T14:29:43,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183434) roll requested 2024-11-13T14:29:43,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183440 2024-11-13T14:29:43,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183440 2024-11-13T14:29:43,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183440) roll requested 2024-11-13T14:29:43,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183448 2024-11-13T14:29:43,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183448 2024-11-13T14:29:43,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183448) roll requested 2024-11-13T14:29:43,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183456 2024-11-13T14:29:43,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183456 2024-11-13T14:29:43,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183456) roll requested 2024-11-13T14:29:43,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183462 2024-11-13T14:29:43,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183462 2024-11-13T14:29:43,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183462) roll requested 2024-11-13T14:29:43,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183468 2024-11-13T14:29:43,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183468 2024-11-13T14:29:43,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183468) roll requested 2024-11-13T14:29:43,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183476 2024-11-13T14:29:43,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183476 2024-11-13T14:29:43,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183476) roll requested 2024-11-13T14:29:43,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183483 2024-11-13T14:29:43,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183483 2024-11-13T14:29:43,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183483) roll requested 2024-11-13T14:29:43,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183489 2024-11-13T14:29:43,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183489 2024-11-13T14:29:43,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183489) roll requested 2024-11-13T14:29:43,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183496 2024-11-13T14:29:43,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183496 2024-11-13T14:29:43,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183496) roll requested 2024-11-13T14:29:43,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183502 2024-11-13T14:29:43,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183502 2024-11-13T14:29:43,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183502) roll requested 2024-11-13T14:29:43,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183507 2024-11-13T14:29:43,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183507 2024-11-13T14:29:43,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183507) roll requested 2024-11-13T14:29:43,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183511 2024-11-13T14:29:43,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183511 2024-11-13T14:29:43,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183511) roll requested 2024-11-13T14:29:43,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183518 2024-11-13T14:29:43,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183518 2024-11-13T14:29:43,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183518) roll requested 2024-11-13T14:29:43,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183525 2024-11-13T14:29:43,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183525 2024-11-13T14:29:43,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183525) roll requested 2024-11-13T14:29:43,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183533 2024-11-13T14:29:43,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183533 2024-11-13T14:29:43,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183533) roll requested 2024-11-13T14:29:43,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183543 2024-11-13T14:29:43,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183543 2024-11-13T14:29:43,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183543) roll requested 2024-11-13T14:29:43,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183552 2024-11-13T14:29:43,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183552 2024-11-13T14:29:43,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183552) roll requested 2024-11-13T14:29:43,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183561 2024-11-13T14:29:43,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183561 2024-11-13T14:29:43,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183561) roll requested 2024-11-13T14:29:43,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183571 2024-11-13T14:29:43,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183571 2024-11-13T14:29:43,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183571) roll requested 2024-11-13T14:29:43,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183582 2024-11-13T14:29:43,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183582 2024-11-13T14:29:43,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183582) roll requested 2024-11-13T14:29:43,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183587 2024-11-13T14:29:43,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183587 2024-11-13T14:29:43,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183587) roll requested 2024-11-13T14:29:43,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183594 2024-11-13T14:29:43,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183594 2024-11-13T14:29:43,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183594) roll requested 2024-11-13T14:29:43,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183602 2024-11-13T14:29:43,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183602 2024-11-13T14:29:43,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183602) roll requested 2024-11-13T14:29:43,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183610 2024-11-13T14:29:43,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183610 2024-11-13T14:29:43,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183610) roll requested 2024-11-13T14:29:43,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183617 2024-11-13T14:29:43,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183617 2024-11-13T14:29:43,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183617) roll requested 2024-11-13T14:29:43,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183623 2024-11-13T14:29:43,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183623 2024-11-13T14:29:43,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183623) roll requested 2024-11-13T14:29:43,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183630 2024-11-13T14:29:43,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183630 2024-11-13T14:29:43,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183630) roll requested 2024-11-13T14:29:43,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183638 2024-11-13T14:29:43,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183638 2024-11-13T14:29:43,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183638) roll requested 2024-11-13T14:29:43,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183646 2024-11-13T14:29:43,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183646 2024-11-13T14:29:43,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183646) roll requested 2024-11-13T14:29:43,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183655 2024-11-13T14:29:43,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183655 2024-11-13T14:29:43,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183655) roll requested 2024-11-13T14:29:43,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183665 2024-11-13T14:29:43,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183665 2024-11-13T14:29:43,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183665) roll requested 2024-11-13T14:29:43,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183673 2024-11-13T14:29:43,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183673 2024-11-13T14:29:43,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183673) roll requested 2024-11-13T14:29:43,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183681 2024-11-13T14:29:43,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183681 2024-11-13T14:29:43,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183681) roll requested 2024-11-13T14:29:43,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183689 2024-11-13T14:29:43,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183689 2024-11-13T14:29:43,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183689) roll requested 2024-11-13T14:29:43,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183698 2024-11-13T14:29:43,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183698 2024-11-13T14:29:43,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183698) roll requested 2024-11-13T14:29:43,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183708 2024-11-13T14:29:43,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183708 2024-11-13T14:29:43,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183708) roll requested 2024-11-13T14:29:43,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183716 2024-11-13T14:29:43,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183716 2024-11-13T14:29:43,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183716) roll requested 2024-11-13T14:29:43,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183725 2024-11-13T14:29:43,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183725 2024-11-13T14:29:43,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183725) roll requested 2024-11-13T14:29:43,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183736 2024-11-13T14:29:43,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183736 2024-11-13T14:29:43,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183736) roll requested 2024-11-13T14:29:43,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183744 2024-11-13T14:29:43,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183744 2024-11-13T14:29:43,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183744) roll requested 2024-11-13T14:29:43,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183753 2024-11-13T14:29:43,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183753 2024-11-13T14:29:43,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183753) roll requested 2024-11-13T14:29:43,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183760 2024-11-13T14:29:43,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183760 2024-11-13T14:29:43,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183760) roll requested 2024-11-13T14:29:43,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183766 2024-11-13T14:29:43,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183766 2024-11-13T14:29:43,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183766) roll requested 2024-11-13T14:29:43,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183774 2024-11-13T14:29:43,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183774 2024-11-13T14:29:43,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183774) roll requested 2024-11-13T14:29:43,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183785 2024-11-13T14:29:43,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183785 2024-11-13T14:29:43,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183785) roll requested 2024-11-13T14:29:43,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183791 2024-11-13T14:29:43,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183791 2024-11-13T14:29:43,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183791) roll requested 2024-11-13T14:29:43,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183797 2024-11-13T14:29:43,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183797 2024-11-13T14:29:43,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183797) roll requested 2024-11-13T14:29:43,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183802 2024-11-13T14:29:43,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183802 2024-11-13T14:29:43,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183802) roll requested 2024-11-13T14:29:43,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183809 2024-11-13T14:29:43,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183809 2024-11-13T14:29:43,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183809) roll requested 2024-11-13T14:29:43,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183816 2024-11-13T14:29:43,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183816 2024-11-13T14:29:43,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183816) roll requested 2024-11-13T14:29:43,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183827 2024-11-13T14:29:43,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183827 2024-11-13T14:29:43,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183827) roll requested 2024-11-13T14:29:43,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183833 2024-11-13T14:29:43,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183833 2024-11-13T14:29:43,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183833) roll requested 2024-11-13T14:29:43,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183839 2024-11-13T14:29:43,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183839 2024-11-13T14:29:43,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183839) roll requested 2024-11-13T14:29:43,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183847 2024-11-13T14:29:43,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183847 2024-11-13T14:29:43,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183847) roll requested 2024-11-13T14:29:43,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183852 2024-11-13T14:29:43,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183852 2024-11-13T14:29:43,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183852) roll requested 2024-11-13T14:29:43,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183858 2024-11-13T14:29:43,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183858 2024-11-13T14:29:43,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183858) roll requested 2024-11-13T14:29:43,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183864 2024-11-13T14:29:43,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183864 2024-11-13T14:29:43,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183864) roll requested 2024-11-13T14:29:43,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183872 2024-11-13T14:29:43,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183872 2024-11-13T14:29:43,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183872) roll requested 2024-11-13T14:29:43,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183878 2024-11-13T14:29:43,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183878 2024-11-13T14:29:43,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183878) roll requested 2024-11-13T14:29:43,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183884 2024-11-13T14:29:43,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183884 2024-11-13T14:29:43,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183884) roll requested 2024-11-13T14:29:43,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183890 2024-11-13T14:29:43,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183890 2024-11-13T14:29:43,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183890) roll requested 2024-11-13T14:29:43,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183900 2024-11-13T14:29:43,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183900 2024-11-13T14:29:43,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183900) roll requested 2024-11-13T14:29:43,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183905 2024-11-13T14:29:43,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183905 2024-11-13T14:29:43,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183905) roll requested 2024-11-13T14:29:43,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183912 2024-11-13T14:29:43,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183912 2024-11-13T14:29:43,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183912) roll requested 2024-11-13T14:29:43,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183918 2024-11-13T14:29:43,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183918 2024-11-13T14:29:43,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183918) roll requested 2024-11-13T14:29:43,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183923 2024-11-13T14:29:43,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183923 2024-11-13T14:29:43,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183923) roll requested 2024-11-13T14:29:43,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183929 2024-11-13T14:29:43,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183929 2024-11-13T14:29:43,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183929) roll requested 2024-11-13T14:29:43,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183937 2024-11-13T14:29:43,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183937 2024-11-13T14:29:43,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183937) roll requested 2024-11-13T14:29:43,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183943 2024-11-13T14:29:43,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183943 2024-11-13T14:29:43,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183943) roll requested 2024-11-13T14:29:43,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183949 2024-11-13T14:29:43,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183949 2024-11-13T14:29:43,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183949) roll requested 2024-11-13T14:29:43,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183955 2024-11-13T14:29:43,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183955 2024-11-13T14:29:43,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183955) roll requested 2024-11-13T14:29:43,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183960 2024-11-13T14:29:43,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183960 2024-11-13T14:29:43,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183960) roll requested 2024-11-13T14:29:43,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183965 2024-11-13T14:29:43,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183965 2024-11-13T14:29:43,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183965) roll requested 2024-11-13T14:29:43,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183973 2024-11-13T14:29:43,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183973 2024-11-13T14:29:43,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183973) roll requested 2024-11-13T14:29:43,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183979 2024-11-13T14:29:43,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183979 2024-11-13T14:29:43,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183979) roll requested 2024-11-13T14:29:43,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183985 2024-11-13T14:29:43,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183985 2024-11-13T14:29:43,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183985) roll requested 2024-11-13T14:29:43,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183991 2024-11-13T14:29:43,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:43,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183991 2024-11-13T14:29:43,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:43,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:43,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:43,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:43,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183991) roll requested 2024-11-13T14:29:43,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508183999 2024-11-13T14:29:44,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183999 2024-11-13T14:29:44,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508183999) roll requested 2024-11-13T14:29:44,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184005 2024-11-13T14:29:44,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508183999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184005 2024-11-13T14:29:44,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184005) roll requested 2024-11-13T14:29:44,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184011 2024-11-13T14:29:44,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184011 2024-11-13T14:29:44,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184011) roll requested 2024-11-13T14:29:44,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184018 2024-11-13T14:29:44,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184018 2024-11-13T14:29:44,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184018) roll requested 2024-11-13T14:29:44,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184025 2024-11-13T14:29:44,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184025 2024-11-13T14:29:44,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184025) roll requested 2024-11-13T14:29:44,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184032 2024-11-13T14:29:44,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184032 2024-11-13T14:29:44,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184032) roll requested 2024-11-13T14:29:44,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184042 2024-11-13T14:29:44,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184042 2024-11-13T14:29:44,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184042) roll requested 2024-11-13T14:29:44,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184051 2024-11-13T14:29:44,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184051 2024-11-13T14:29:44,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184051) roll requested 2024-11-13T14:29:44,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184059 2024-11-13T14:29:44,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184059 2024-11-13T14:29:44,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184059) roll requested 2024-11-13T14:29:44,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184067 2024-11-13T14:29:44,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184067 2024-11-13T14:29:44,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184067) roll requested 2024-11-13T14:29:44,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184076 2024-11-13T14:29:44,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184076 2024-11-13T14:29:44,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184076) roll requested 2024-11-13T14:29:44,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184084 2024-11-13T14:29:44,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184084 2024-11-13T14:29:44,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184084) roll requested 2024-11-13T14:29:44,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184093 2024-11-13T14:29:44,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184093 2024-11-13T14:29:44,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184093) roll requested 2024-11-13T14:29:44,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184101 2024-11-13T14:29:44,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184101 2024-11-13T14:29:44,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184101) roll requested 2024-11-13T14:29:44,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184106 2024-11-13T14:29:44,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184106 2024-11-13T14:29:44,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184106) roll requested 2024-11-13T14:29:44,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184114 2024-11-13T14:29:44,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184114 2024-11-13T14:29:44,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184114) roll requested 2024-11-13T14:29:44,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184120 2024-11-13T14:29:44,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184120 2024-11-13T14:29:44,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184120) roll requested 2024-11-13T14:29:44,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184125 2024-11-13T14:29:44,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184125 2024-11-13T14:29:44,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184125) roll requested 2024-11-13T14:29:44,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184129 2024-11-13T14:29:44,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184129 2024-11-13T14:29:44,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184129) roll requested 2024-11-13T14:29:44,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184135 2024-11-13T14:29:44,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184135 2024-11-13T14:29:44,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184135) roll requested 2024-11-13T14:29:44,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184141 2024-11-13T14:29:44,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184141 2024-11-13T14:29:44,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184141) roll requested 2024-11-13T14:29:44,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184149 2024-11-13T14:29:44,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184149 2024-11-13T14:29:44,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184149) roll requested 2024-11-13T14:29:44,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184157 2024-11-13T14:29:44,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184157 2024-11-13T14:29:44,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184157) roll requested 2024-11-13T14:29:44,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184165 2024-11-13T14:29:44,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184165 2024-11-13T14:29:44,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184165) roll requested 2024-11-13T14:29:44,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184174 2024-11-13T14:29:44,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184174 2024-11-13T14:29:44,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184174) roll requested 2024-11-13T14:29:44,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184184 2024-11-13T14:29:44,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184184 2024-11-13T14:29:44,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184184) roll requested 2024-11-13T14:29:44,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184194 2024-11-13T14:29:44,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184194 2024-11-13T14:29:44,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184194) roll requested 2024-11-13T14:29:44,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184203 2024-11-13T14:29:44,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184203 2024-11-13T14:29:44,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184203) roll requested 2024-11-13T14:29:44,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184211 2024-11-13T14:29:44,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184211 2024-11-13T14:29:44,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184211) roll requested 2024-11-13T14:29:44,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184220 2024-11-13T14:29:44,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184220 2024-11-13T14:29:44,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184220) roll requested 2024-11-13T14:29:44,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184229 2024-11-13T14:29:44,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184229 2024-11-13T14:29:44,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184229) roll requested 2024-11-13T14:29:44,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184241 2024-11-13T14:29:44,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184241 2024-11-13T14:29:44,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184241) roll requested 2024-11-13T14:29:44,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184248 2024-11-13T14:29:44,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184248 2024-11-13T14:29:44,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184248) roll requested 2024-11-13T14:29:44,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184255 2024-11-13T14:29:44,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184255 2024-11-13T14:29:44,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184255) roll requested 2024-11-13T14:29:44,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184263 2024-11-13T14:29:44,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184263 2024-11-13T14:29:44,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184263) roll requested 2024-11-13T14:29:44,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184271 2024-11-13T14:29:44,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184271 2024-11-13T14:29:44,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184271) roll requested 2024-11-13T14:29:44,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184276 2024-11-13T14:29:44,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184276 2024-11-13T14:29:44,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184276) roll requested 2024-11-13T14:29:44,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184282 2024-11-13T14:29:44,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184282 2024-11-13T14:29:44,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184282) roll requested 2024-11-13T14:29:44,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184288 2024-11-13T14:29:44,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184288 2024-11-13T14:29:44,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184288) roll requested 2024-11-13T14:29:44,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184294 2024-11-13T14:29:44,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184294 2024-11-13T14:29:44,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184294) roll requested 2024-11-13T14:29:44,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184303 2024-11-13T14:29:44,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184303 2024-11-13T14:29:44,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184303) roll requested 2024-11-13T14:29:44,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184309 2024-11-13T14:29:44,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184309 2024-11-13T14:29:44,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184309) roll requested 2024-11-13T14:29:44,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184314 2024-11-13T14:29:44,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184314 2024-11-13T14:29:44,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184314) roll requested 2024-11-13T14:29:44,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184321 2024-11-13T14:29:44,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184321 2024-11-13T14:29:44,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184321) roll requested 2024-11-13T14:29:44,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184327 2024-11-13T14:29:44,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184327 2024-11-13T14:29:44,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184327) roll requested 2024-11-13T14:29:44,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184332 2024-11-13T14:29:44,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184332 2024-11-13T14:29:44,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184332) roll requested 2024-11-13T14:29:44,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184338 2024-11-13T14:29:44,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184338 2024-11-13T14:29:44,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184338) roll requested 2024-11-13T14:29:44,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184344 2024-11-13T14:29:44,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184344 2024-11-13T14:29:44,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184344) roll requested 2024-11-13T14:29:44,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184349 2024-11-13T14:29:44,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184349 2024-11-13T14:29:44,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184349) roll requested 2024-11-13T14:29:44,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184355 2024-11-13T14:29:44,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184355 2024-11-13T14:29:44,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184355) roll requested 2024-11-13T14:29:44,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184363 2024-11-13T14:29:44,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184363 2024-11-13T14:29:44,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184363) roll requested 2024-11-13T14:29:44,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184371 2024-11-13T14:29:44,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184371 2024-11-13T14:29:44,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184371) roll requested 2024-11-13T14:29:44,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184378 2024-11-13T14:29:44,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184378 2024-11-13T14:29:44,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184378) roll requested 2024-11-13T14:29:44,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184384 2024-11-13T14:29:44,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184384 2024-11-13T14:29:44,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184384) roll requested 2024-11-13T14:29:44,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184391 2024-11-13T14:29:44,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184391 2024-11-13T14:29:44,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184391) roll requested 2024-11-13T14:29:44,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184401 2024-11-13T14:29:44,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184401 2024-11-13T14:29:44,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184401) roll requested 2024-11-13T14:29:44,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184411 2024-11-13T14:29:44,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184411 2024-11-13T14:29:44,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184411) roll requested 2024-11-13T14:29:44,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184418 2024-11-13T14:29:44,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184418 2024-11-13T14:29:44,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184418) roll requested 2024-11-13T14:29:44,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184427 2024-11-13T14:29:44,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184427 2024-11-13T14:29:44,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184427) roll requested 2024-11-13T14:29:44,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184437 2024-11-13T14:29:44,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184437 2024-11-13T14:29:44,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184437) roll requested 2024-11-13T14:29:44,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184445 2024-11-13T14:29:44,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184445 2024-11-13T14:29:44,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184445) roll requested 2024-11-13T14:29:44,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184456 2024-11-13T14:29:44,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184456 2024-11-13T14:29:44,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184456) roll requested 2024-11-13T14:29:44,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184466 2024-11-13T14:29:44,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184466 2024-11-13T14:29:44,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184466) roll requested 2024-11-13T14:29:44,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184473 2024-11-13T14:29:44,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184473 2024-11-13T14:29:44,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184473) roll requested 2024-11-13T14:29:44,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184482 2024-11-13T14:29:44,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184482 2024-11-13T14:29:44,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184482) roll requested 2024-11-13T14:29:44,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184489 2024-11-13T14:29:44,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184489 2024-11-13T14:29:44,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184489) roll requested 2024-11-13T14:29:44,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184504 2024-11-13T14:29:44,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184504 2024-11-13T14:29:44,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184504) roll requested 2024-11-13T14:29:44,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184511 2024-11-13T14:29:44,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184511 2024-11-13T14:29:44,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184511) roll requested 2024-11-13T14:29:44,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184519 2024-11-13T14:29:44,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184519 2024-11-13T14:29:44,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184519) roll requested 2024-11-13T14:29:44,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184531 2024-11-13T14:29:44,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184531 2024-11-13T14:29:44,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184531) roll requested 2024-11-13T14:29:44,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184541 2024-11-13T14:29:44,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184541 2024-11-13T14:29:44,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184541) roll requested 2024-11-13T14:29:44,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184549 2024-11-13T14:29:44,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184549 2024-11-13T14:29:44,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184549) roll requested 2024-11-13T14:29:44,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184554 2024-11-13T14:29:44,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184554 2024-11-13T14:29:44,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184554) roll requested 2024-11-13T14:29:44,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184563 2024-11-13T14:29:44,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184563 2024-11-13T14:29:44,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184563) roll requested 2024-11-13T14:29:44,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184569 2024-11-13T14:29:44,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184569 2024-11-13T14:29:44,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184569) roll requested 2024-11-13T14:29:44,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184574 2024-11-13T14:29:44,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184574 2024-11-13T14:29:44,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184574) roll requested 2024-11-13T14:29:44,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184581 2024-11-13T14:29:44,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184581 2024-11-13T14:29:44,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184581) roll requested 2024-11-13T14:29:44,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184588 2024-11-13T14:29:44,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184588 2024-11-13T14:29:44,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184588) roll requested 2024-11-13T14:29:44,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184595 2024-11-13T14:29:44,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184595 2024-11-13T14:29:44,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184595) roll requested 2024-11-13T14:29:44,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184602 2024-11-13T14:29:44,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184602 2024-11-13T14:29:44,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184602) roll requested 2024-11-13T14:29:44,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184607 2024-11-13T14:29:44,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184607 2024-11-13T14:29:44,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184607) roll requested 2024-11-13T14:29:44,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184613 2024-11-13T14:29:44,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184613 2024-11-13T14:29:44,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184613) roll requested 2024-11-13T14:29:44,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184620 2024-11-13T14:29:44,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184620 2024-11-13T14:29:44,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184620) roll requested 2024-11-13T14:29:44,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184629 2024-11-13T14:29:44,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184629 2024-11-13T14:29:44,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184629) roll requested 2024-11-13T14:29:44,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184634 2024-11-13T14:29:44,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184634 2024-11-13T14:29:44,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184634) roll requested 2024-11-13T14:29:44,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184640 2024-11-13T14:29:44,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184640 2024-11-13T14:29:44,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184640) roll requested 2024-11-13T14:29:44,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184646 2024-11-13T14:29:44,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184646 2024-11-13T14:29:44,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184646) roll requested 2024-11-13T14:29:44,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184651 2024-11-13T14:29:44,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184651 2024-11-13T14:29:44,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184651) roll requested 2024-11-13T14:29:44,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184657 2024-11-13T14:29:44,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184657 2024-11-13T14:29:44,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184657) roll requested 2024-11-13T14:29:44,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184664 2024-11-13T14:29:44,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184664 2024-11-13T14:29:44,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184664) roll requested 2024-11-13T14:29:44,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184672 2024-11-13T14:29:44,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184672 2024-11-13T14:29:44,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184672) roll requested 2024-11-13T14:29:44,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184679 2024-11-13T14:29:44,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184679 2024-11-13T14:29:44,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184679) roll requested 2024-11-13T14:29:44,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184684 2024-11-13T14:29:44,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184684 2024-11-13T14:29:44,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184684) roll requested 2024-11-13T14:29:44,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184689 2024-11-13T14:29:44,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184689 2024-11-13T14:29:44,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184689) roll requested 2024-11-13T14:29:44,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184695 2024-11-13T14:29:44,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184695 2024-11-13T14:29:44,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184695) roll requested 2024-11-13T14:29:44,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184701 2024-11-13T14:29:44,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184701 2024-11-13T14:29:44,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184701) roll requested 2024-11-13T14:29:44,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184708 2024-11-13T14:29:44,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184708 2024-11-13T14:29:44,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184708) roll requested 2024-11-13T14:29:44,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184715 2024-11-13T14:29:44,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184715 2024-11-13T14:29:44,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184715) roll requested 2024-11-13T14:29:44,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184724 2024-11-13T14:29:44,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184724 2024-11-13T14:29:44,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184724) roll requested 2024-11-13T14:29:44,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184732 2024-11-13T14:29:44,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184732 2024-11-13T14:29:44,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184732) roll requested 2024-11-13T14:29:44,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184741 2024-11-13T14:29:44,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184741 2024-11-13T14:29:44,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184741) roll requested 2024-11-13T14:29:44,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184749 2024-11-13T14:29:44,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184749 2024-11-13T14:29:44,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184749) roll requested 2024-11-13T14:29:44,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184757 2024-11-13T14:29:44,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184757 2024-11-13T14:29:44,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184757) roll requested 2024-11-13T14:29:44,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184766 2024-11-13T14:29:44,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184766 2024-11-13T14:29:44,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184766) roll requested 2024-11-13T14:29:44,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184775 2024-11-13T14:29:44,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184775 2024-11-13T14:29:44,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184775) roll requested 2024-11-13T14:29:44,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184784 2024-11-13T14:29:44,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184784 2024-11-13T14:29:44,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184784) roll requested 2024-11-13T14:29:44,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184789 2024-11-13T14:29:44,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184789 2024-11-13T14:29:44,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184789) roll requested 2024-11-13T14:29:44,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184795 2024-11-13T14:29:44,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184795 2024-11-13T14:29:44,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184795) roll requested 2024-11-13T14:29:44,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184802 2024-11-13T14:29:44,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184802 2024-11-13T14:29:44,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184802) roll requested 2024-11-13T14:29:44,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184810 2024-11-13T14:29:44,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184810 2024-11-13T14:29:44,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184810) roll requested 2024-11-13T14:29:44,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184819 2024-11-13T14:29:44,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184819 2024-11-13T14:29:44,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184819) roll requested 2024-11-13T14:29:44,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184827 2024-11-13T14:29:44,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184827 2024-11-13T14:29:44,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184827) roll requested 2024-11-13T14:29:44,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184836 2024-11-13T14:29:44,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184836 2024-11-13T14:29:44,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184836) roll requested 2024-11-13T14:29:44,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184844 2024-11-13T14:29:44,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184844 2024-11-13T14:29:44,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184844) roll requested 2024-11-13T14:29:44,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184852 2024-11-13T14:29:44,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184852 2024-11-13T14:29:44,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184852) roll requested 2024-11-13T14:29:44,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184862 2024-11-13T14:29:44,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184862 2024-11-13T14:29:44,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184862) roll requested 2024-11-13T14:29:44,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184871 2024-11-13T14:29:44,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184871 2024-11-13T14:29:44,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184871) roll requested 2024-11-13T14:29:44,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184879 2024-11-13T14:29:44,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184879 2024-11-13T14:29:44,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184879) roll requested 2024-11-13T14:29:44,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184884 2024-11-13T14:29:44,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184884 2024-11-13T14:29:44,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184884) roll requested 2024-11-13T14:29:44,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184890 2024-11-13T14:29:44,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184890 2024-11-13T14:29:44,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184890) roll requested 2024-11-13T14:29:44,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184898 2024-11-13T14:29:44,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184898 2024-11-13T14:29:44,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184898) roll requested 2024-11-13T14:29:44,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184906 2024-11-13T14:29:44,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184906 2024-11-13T14:29:44,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184906) roll requested 2024-11-13T14:29:44,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184915 2024-11-13T14:29:44,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184915 2024-11-13T14:29:44,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184915) roll requested 2024-11-13T14:29:44,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184922 2024-11-13T14:29:44,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184922 2024-11-13T14:29:44,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184922) roll requested 2024-11-13T14:29:44,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184929 2024-11-13T14:29:44,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184929 2024-11-13T14:29:44,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184929) roll requested 2024-11-13T14:29:44,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184938 2024-11-13T14:29:44,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184938 2024-11-13T14:29:44,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184938) roll requested 2024-11-13T14:29:44,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184948 2024-11-13T14:29:44,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184948 2024-11-13T14:29:44,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184948) roll requested 2024-11-13T14:29:44,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184958 2024-11-13T14:29:44,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184958 2024-11-13T14:29:44,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184958) roll requested 2024-11-13T14:29:44,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184968 2024-11-13T14:29:44,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184968 2024-11-13T14:29:44,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184968) roll requested 2024-11-13T14:29:44,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184979 2024-11-13T14:29:44,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184979 2024-11-13T14:29:44,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184979) roll requested 2024-11-13T14:29:44,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184987 2024-11-13T14:29:44,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:44,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184987 2024-11-13T14:29:44,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:44,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:44,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:44,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:44,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184987) roll requested 2024-11-13T14:29:44,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508184995 2024-11-13T14:29:45,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184995 2024-11-13T14:29:45,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508184995) roll requested 2024-11-13T14:29:45,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185004 2024-11-13T14:29:45,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508184995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185004 2024-11-13T14:29:45,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185004) roll requested 2024-11-13T14:29:45,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185012 2024-11-13T14:29:45,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185012 2024-11-13T14:29:45,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185012) roll requested 2024-11-13T14:29:45,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185018 2024-11-13T14:29:45,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185018 2024-11-13T14:29:45,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185018) roll requested 2024-11-13T14:29:45,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185023 2024-11-13T14:29:45,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185023 2024-11-13T14:29:45,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185023) roll requested 2024-11-13T14:29:45,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185031 2024-11-13T14:29:45,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185031 2024-11-13T14:29:45,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185031) roll requested 2024-11-13T14:29:45,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185039 2024-11-13T14:29:45,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185039 2024-11-13T14:29:45,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185039) roll requested 2024-11-13T14:29:45,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185048 2024-11-13T14:29:45,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185048 2024-11-13T14:29:45,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185048) roll requested 2024-11-13T14:29:45,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185057 2024-11-13T14:29:45,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185057 2024-11-13T14:29:45,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185057) roll requested 2024-11-13T14:29:45,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185066 2024-11-13T14:29:45,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185066 2024-11-13T14:29:45,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185066) roll requested 2024-11-13T14:29:45,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185076 2024-11-13T14:29:45,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185076 2024-11-13T14:29:45,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185076) roll requested 2024-11-13T14:29:45,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185084 2024-11-13T14:29:45,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185084 2024-11-13T14:29:45,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185084) roll requested 2024-11-13T14:29:45,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185093 2024-11-13T14:29:45,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185093 2024-11-13T14:29:45,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185093) roll requested 2024-11-13T14:29:45,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185098 2024-11-13T14:29:45,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185098 2024-11-13T14:29:45,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185098) roll requested 2024-11-13T14:29:45,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185103 2024-11-13T14:29:45,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185103 2024-11-13T14:29:45,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185103) roll requested 2024-11-13T14:29:45,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185110 2024-11-13T14:29:45,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185110 2024-11-13T14:29:45,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185110) roll requested 2024-11-13T14:29:45,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185117 2024-11-13T14:29:45,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185117 2024-11-13T14:29:45,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185117) roll requested 2024-11-13T14:29:45,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185124 2024-11-13T14:29:45,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185124 2024-11-13T14:29:45,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185124) roll requested 2024-11-13T14:29:45,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185132 2024-11-13T14:29:45,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185132 2024-11-13T14:29:45,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185132) roll requested 2024-11-13T14:29:45,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185140 2024-11-13T14:29:45,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185140 2024-11-13T14:29:45,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185140) roll requested 2024-11-13T14:29:45,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185148 2024-11-13T14:29:45,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185148 2024-11-13T14:29:45,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185148) roll requested 2024-11-13T14:29:45,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185155 2024-11-13T14:29:45,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185155 2024-11-13T14:29:45,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185155) roll requested 2024-11-13T14:29:45,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185163 2024-11-13T14:29:45,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185163 2024-11-13T14:29:45,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185163) roll requested 2024-11-13T14:29:45,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185171 2024-11-13T14:29:45,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185171 2024-11-13T14:29:45,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185171) roll requested 2024-11-13T14:29:45,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185177 2024-11-13T14:29:45,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185177 2024-11-13T14:29:45,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185177) roll requested 2024-11-13T14:29:45,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185185 2024-11-13T14:29:45,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185185 2024-11-13T14:29:45,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185185) roll requested 2024-11-13T14:29:45,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185192 2024-11-13T14:29:45,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185192 2024-11-13T14:29:45,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185192) roll requested 2024-11-13T14:29:45,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185200 2024-11-13T14:29:45,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185200 2024-11-13T14:29:45,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185200) roll requested 2024-11-13T14:29:45,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185208 2024-11-13T14:29:45,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185208 2024-11-13T14:29:45,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185208) roll requested 2024-11-13T14:29:45,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185216 2024-11-13T14:29:45,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185216 2024-11-13T14:29:45,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185216) roll requested 2024-11-13T14:29:45,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185223 2024-11-13T14:29:45,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185223 2024-11-13T14:29:45,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185223) roll requested 2024-11-13T14:29:45,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185230 2024-11-13T14:29:45,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185230 2024-11-13T14:29:45,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185230) roll requested 2024-11-13T14:29:45,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185238 2024-11-13T14:29:45,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185238 2024-11-13T14:29:45,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185238) roll requested 2024-11-13T14:29:45,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185247 2024-11-13T14:29:45,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185247 2024-11-13T14:29:45,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185247) roll requested 2024-11-13T14:29:45,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185257 2024-11-13T14:29:45,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185257 2024-11-13T14:29:45,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185257) roll requested 2024-11-13T14:29:45,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185267 2024-11-13T14:29:45,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185267 2024-11-13T14:29:45,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185267) roll requested 2024-11-13T14:29:45,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185277 2024-11-13T14:29:45,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185277 2024-11-13T14:29:45,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185277) roll requested 2024-11-13T14:29:45,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185286 2024-11-13T14:29:45,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185286 2024-11-13T14:29:45,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185286) roll requested 2024-11-13T14:29:45,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185294 2024-11-13T14:29:45,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185294 2024-11-13T14:29:45,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185294) roll requested 2024-11-13T14:29:45,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185302 2024-11-13T14:29:45,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185302 2024-11-13T14:29:45,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185302) roll requested 2024-11-13T14:29:45,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185311 2024-11-13T14:29:45,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185311 2024-11-13T14:29:45,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185311) roll requested 2024-11-13T14:29:45,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185321 2024-11-13T14:29:45,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185321 2024-11-13T14:29:45,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185321) roll requested 2024-11-13T14:29:45,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185329 2024-11-13T14:29:45,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185329 2024-11-13T14:29:45,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185329) roll requested 2024-11-13T14:29:45,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185337 2024-11-13T14:29:45,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185337 2024-11-13T14:29:45,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185337) roll requested 2024-11-13T14:29:45,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185347 2024-11-13T14:29:45,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185347 2024-11-13T14:29:45,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185347) roll requested 2024-11-13T14:29:45,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185355 2024-11-13T14:29:45,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185355 2024-11-13T14:29:45,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185355) roll requested 2024-11-13T14:29:45,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185363 2024-11-13T14:29:45,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185363 2024-11-13T14:29:45,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185363) roll requested 2024-11-13T14:29:45,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185371 2024-11-13T14:29:45,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185371 2024-11-13T14:29:45,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185371) roll requested 2024-11-13T14:29:45,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185380 2024-11-13T14:29:45,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185380 2024-11-13T14:29:45,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185380) roll requested 2024-11-13T14:29:45,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185391 2024-11-13T14:29:45,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185391 2024-11-13T14:29:45,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185391) roll requested 2024-11-13T14:29:45,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185402 2024-11-13T14:29:45,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185402 2024-11-13T14:29:45,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185402) roll requested 2024-11-13T14:29:45,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185408 2024-11-13T14:29:45,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185408 2024-11-13T14:29:45,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185408) roll requested 2024-11-13T14:29:45,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185415 2024-11-13T14:29:45,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185415 2024-11-13T14:29:45,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185415) roll requested 2024-11-13T14:29:45,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185423 2024-11-13T14:29:45,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185423 2024-11-13T14:29:45,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185423) roll requested 2024-11-13T14:29:45,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185432 2024-11-13T14:29:45,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185432 2024-11-13T14:29:45,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185432) roll requested 2024-11-13T14:29:45,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185441 2024-11-13T14:29:45,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185441 2024-11-13T14:29:45,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185441) roll requested 2024-11-13T14:29:45,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185449 2024-11-13T14:29:45,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185449 2024-11-13T14:29:45,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185449) roll requested 2024-11-13T14:29:45,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185457 2024-11-13T14:29:45,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185457 2024-11-13T14:29:45,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185457) roll requested 2024-11-13T14:29:45,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185463 2024-11-13T14:29:45,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185463 2024-11-13T14:29:45,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185463) roll requested 2024-11-13T14:29:45,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185469 2024-11-13T14:29:45,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185469 2024-11-13T14:29:45,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185469) roll requested 2024-11-13T14:29:45,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185476 2024-11-13T14:29:45,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185476 2024-11-13T14:29:45,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185476) roll requested 2024-11-13T14:29:45,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185484 2024-11-13T14:29:45,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185484 2024-11-13T14:29:45,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185484) roll requested 2024-11-13T14:29:45,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185492 2024-11-13T14:29:45,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185492 2024-11-13T14:29:45,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185492) roll requested 2024-11-13T14:29:45,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185499 2024-11-13T14:29:45,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185499 2024-11-13T14:29:45,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185499) roll requested 2024-11-13T14:29:45,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185507 2024-11-13T14:29:45,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185507 2024-11-13T14:29:45,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185507) roll requested 2024-11-13T14:29:45,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185514 2024-11-13T14:29:45,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185514 2024-11-13T14:29:45,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185514) roll requested 2024-11-13T14:29:45,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185521 2024-11-13T14:29:45,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185521 2024-11-13T14:29:45,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185521) roll requested 2024-11-13T14:29:45,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185527 2024-11-13T14:29:45,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185527 2024-11-13T14:29:45,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185527) roll requested 2024-11-13T14:29:45,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185534 2024-11-13T14:29:45,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185534 2024-11-13T14:29:45,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185534) roll requested 2024-11-13T14:29:45,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185545 2024-11-13T14:29:45,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185545 2024-11-13T14:29:45,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185545) roll requested 2024-11-13T14:29:45,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185552 2024-11-13T14:29:45,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185552 2024-11-13T14:29:45,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185552) roll requested 2024-11-13T14:29:45,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185558 2024-11-13T14:29:45,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185558 2024-11-13T14:29:45,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185558) roll requested 2024-11-13T14:29:45,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185563 2024-11-13T14:29:45,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185563 2024-11-13T14:29:45,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185563) roll requested 2024-11-13T14:29:45,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185570 2024-11-13T14:29:45,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185570 2024-11-13T14:29:45,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185570) roll requested 2024-11-13T14:29:45,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185579 2024-11-13T14:29:45,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185579 2024-11-13T14:29:45,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185579) roll requested 2024-11-13T14:29:45,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185586 2024-11-13T14:29:45,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185586 2024-11-13T14:29:45,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185586) roll requested 2024-11-13T14:29:45,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185593 2024-11-13T14:29:45,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185593 2024-11-13T14:29:45,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185593) roll requested 2024-11-13T14:29:45,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185602 2024-11-13T14:29:45,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185602 2024-11-13T14:29:45,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185602) roll requested 2024-11-13T14:29:45,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185610 2024-11-13T14:29:45,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185610 2024-11-13T14:29:45,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185610) roll requested 2024-11-13T14:29:45,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185615 2024-11-13T14:29:45,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185615 2024-11-13T14:29:45,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185615) roll requested 2024-11-13T14:29:45,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185621 2024-11-13T14:29:45,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185621 2024-11-13T14:29:45,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185621) roll requested 2024-11-13T14:29:45,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185627 2024-11-13T14:29:45,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185627 2024-11-13T14:29:45,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185627) roll requested 2024-11-13T14:29:45,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185634 2024-11-13T14:29:45,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185634 2024-11-13T14:29:45,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185634) roll requested 2024-11-13T14:29:45,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185642 2024-11-13T14:29:45,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185642 2024-11-13T14:29:45,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185642) roll requested 2024-11-13T14:29:45,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185651 2024-11-13T14:29:45,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185651 2024-11-13T14:29:45,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185651) roll requested 2024-11-13T14:29:45,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185660 2024-11-13T14:29:45,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185660 2024-11-13T14:29:45,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185660) roll requested 2024-11-13T14:29:45,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185670 2024-11-13T14:29:45,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185670 2024-11-13T14:29:45,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185670) roll requested 2024-11-13T14:29:45,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185730 2024-11-13T14:29:45,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185730 2024-11-13T14:29:45,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185730) roll requested 2024-11-13T14:29:45,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185738 2024-11-13T14:29:45,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185738 2024-11-13T14:29:45,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185738) roll requested 2024-11-13T14:29:45,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185745 2024-11-13T14:29:45,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185745 2024-11-13T14:29:45,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185745) roll requested 2024-11-13T14:29:45,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185751 2024-11-13T14:29:45,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185751 2024-11-13T14:29:45,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185751) roll requested 2024-11-13T14:29:45,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185759 2024-11-13T14:29:45,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185759 2024-11-13T14:29:45,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185759) roll requested 2024-11-13T14:29:45,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185767 2024-11-13T14:29:45,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185767 2024-11-13T14:29:45,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185767) roll requested 2024-11-13T14:29:45,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185773 2024-11-13T14:29:45,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185773 2024-11-13T14:29:45,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185773) roll requested 2024-11-13T14:29:45,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185781 2024-11-13T14:29:45,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185781 2024-11-13T14:29:45,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185781) roll requested 2024-11-13T14:29:45,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185789 2024-11-13T14:29:45,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185789 2024-11-13T14:29:45,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185789) roll requested 2024-11-13T14:29:45,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185797 2024-11-13T14:29:45,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185797 2024-11-13T14:29:45,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185797) roll requested 2024-11-13T14:29:45,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185803 2024-11-13T14:29:45,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185803 2024-11-13T14:29:45,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185803) roll requested 2024-11-13T14:29:45,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185809 2024-11-13T14:29:45,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185809 2024-11-13T14:29:45,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185809) roll requested 2024-11-13T14:29:45,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185818 2024-11-13T14:29:45,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185818 2024-11-13T14:29:45,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185818) roll requested 2024-11-13T14:29:45,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185827 2024-11-13T14:29:45,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185827 2024-11-13T14:29:45,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185827) roll requested 2024-11-13T14:29:45,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185836 2024-11-13T14:29:45,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185836 2024-11-13T14:29:45,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185836) roll requested 2024-11-13T14:29:45,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185845 2024-11-13T14:29:45,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185845 2024-11-13T14:29:45,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185845) roll requested 2024-11-13T14:29:45,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185855 2024-11-13T14:29:45,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185855 2024-11-13T14:29:45,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185855) roll requested 2024-11-13T14:29:45,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185865 2024-11-13T14:29:45,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185865 2024-11-13T14:29:45,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185865) roll requested 2024-11-13T14:29:45,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185873 2024-11-13T14:29:45,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185873 2024-11-13T14:29:45,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185873) roll requested 2024-11-13T14:29:45,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185880 2024-11-13T14:29:45,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185880 2024-11-13T14:29:45,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185880) roll requested 2024-11-13T14:29:45,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185888 2024-11-13T14:29:45,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185888 2024-11-13T14:29:45,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185888) roll requested 2024-11-13T14:29:45,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185894 2024-11-13T14:29:45,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185894 2024-11-13T14:29:45,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185894) roll requested 2024-11-13T14:29:45,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185900 2024-11-13T14:29:45,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185900 2024-11-13T14:29:45,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185900) roll requested 2024-11-13T14:29:45,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185908 2024-11-13T14:29:45,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185908 2024-11-13T14:29:45,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185908) roll requested 2024-11-13T14:29:45,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185915 2024-11-13T14:29:45,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185915 2024-11-13T14:29:45,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185915) roll requested 2024-11-13T14:29:45,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185924 2024-11-13T14:29:45,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185924 2024-11-13T14:29:45,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185924) roll requested 2024-11-13T14:29:45,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185935 2024-11-13T14:29:45,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185935 2024-11-13T14:29:45,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185935) roll requested 2024-11-13T14:29:45,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185945 2024-11-13T14:29:45,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185945 2024-11-13T14:29:45,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185945) roll requested 2024-11-13T14:29:45,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185954 2024-11-13T14:29:45,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185954 2024-11-13T14:29:45,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185954) roll requested 2024-11-13T14:29:45,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185965 2024-11-13T14:29:45,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185965 2024-11-13T14:29:45,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185965) roll requested 2024-11-13T14:29:45,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185970 2024-11-13T14:29:45,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185970 2024-11-13T14:29:45,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185970) roll requested 2024-11-13T14:29:45,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185975 2024-11-13T14:29:45,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185975 2024-11-13T14:29:45,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185975) roll requested 2024-11-13T14:29:45,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185982 2024-11-13T14:29:45,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185982 2024-11-13T14:29:45,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185982) roll requested 2024-11-13T14:29:45,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185989 2024-11-13T14:29:45,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:45,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185989 2024-11-13T14:29:45,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:45,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:45,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:45,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:45,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185989) roll requested 2024-11-13T14:29:45,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508185998 2024-11-13T14:29:46,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185998 2024-11-13T14:29:46,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508185998) roll requested 2024-11-13T14:29:46,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186007 2024-11-13T14:29:46,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508185998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186007 2024-11-13T14:29:46,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186007) roll requested 2024-11-13T14:29:46,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186017 2024-11-13T14:29:46,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186017 2024-11-13T14:29:46,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186017) roll requested 2024-11-13T14:29:46,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186026 2024-11-13T14:29:46,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186026 2024-11-13T14:29:46,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186026) roll requested 2024-11-13T14:29:46,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186035 2024-11-13T14:29:46,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186035 2024-11-13T14:29:46,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186035) roll requested 2024-11-13T14:29:46,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186044 2024-11-13T14:29:46,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186044 2024-11-13T14:29:46,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186044) roll requested 2024-11-13T14:29:46,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186053 2024-11-13T14:29:46,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186053 2024-11-13T14:29:46,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186053) roll requested 2024-11-13T14:29:46,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186060 2024-11-13T14:29:46,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186060 2024-11-13T14:29:46,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186060) roll requested 2024-11-13T14:29:46,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186067 2024-11-13T14:29:46,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186067 2024-11-13T14:29:46,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186067) roll requested 2024-11-13T14:29:46,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186075 2024-11-13T14:29:46,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186075 2024-11-13T14:29:46,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186075) roll requested 2024-11-13T14:29:46,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186082 2024-11-13T14:29:46,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186082 2024-11-13T14:29:46,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186082) roll requested 2024-11-13T14:29:46,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186091 2024-11-13T14:29:46,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186091 2024-11-13T14:29:46,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186091) roll requested 2024-11-13T14:29:46,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186100 2024-11-13T14:29:46,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186100 2024-11-13T14:29:46,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186100) roll requested 2024-11-13T14:29:46,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186109 2024-11-13T14:29:46,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186109 2024-11-13T14:29:46,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186109) roll requested 2024-11-13T14:29:46,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186118 2024-11-13T14:29:46,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186118 2024-11-13T14:29:46,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186118) roll requested 2024-11-13T14:29:46,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186128 2024-11-13T14:29:46,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186128 2024-11-13T14:29:46,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186128) roll requested 2024-11-13T14:29:46,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186139 2024-11-13T14:29:46,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186139 2024-11-13T14:29:46,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186139) roll requested 2024-11-13T14:29:46,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186148 2024-11-13T14:29:46,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186148 2024-11-13T14:29:46,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186148) roll requested 2024-11-13T14:29:46,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186155 2024-11-13T14:29:46,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186155 2024-11-13T14:29:46,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186155) roll requested 2024-11-13T14:29:46,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186163 2024-11-13T14:29:46,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186163 2024-11-13T14:29:46,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186163) roll requested 2024-11-13T14:29:46,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186170 2024-11-13T14:29:46,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186170 2024-11-13T14:29:46,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186170) roll requested 2024-11-13T14:29:46,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186179 2024-11-13T14:29:46,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186179 2024-11-13T14:29:46,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186179) roll requested 2024-11-13T14:29:46,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186187 2024-11-13T14:29:46,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186187 2024-11-13T14:29:46,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186187) roll requested 2024-11-13T14:29:46,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186195 2024-11-13T14:29:46,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186195 2024-11-13T14:29:46,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186195) roll requested 2024-11-13T14:29:46,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186204 2024-11-13T14:29:46,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186204 2024-11-13T14:29:46,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186204) roll requested 2024-11-13T14:29:46,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186210 2024-11-13T14:29:46,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186210 2024-11-13T14:29:46,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186210) roll requested 2024-11-13T14:29:46,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186216 2024-11-13T14:29:46,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186216 2024-11-13T14:29:46,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186216) roll requested 2024-11-13T14:29:46,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186225 2024-11-13T14:29:46,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186225 2024-11-13T14:29:46,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186225) roll requested 2024-11-13T14:29:46,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186231 2024-11-13T14:29:46,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186231 2024-11-13T14:29:46,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186231) roll requested 2024-11-13T14:29:46,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186236 2024-11-13T14:29:46,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186236 2024-11-13T14:29:46,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186236) roll requested 2024-11-13T14:29:46,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186241 2024-11-13T14:29:46,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186241 2024-11-13T14:29:46,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186241) roll requested 2024-11-13T14:29:46,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186248 2024-11-13T14:29:46,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186248 2024-11-13T14:29:46,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186248) roll requested 2024-11-13T14:29:46,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186255 2024-11-13T14:29:46,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186255 2024-11-13T14:29:46,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186255) roll requested 2024-11-13T14:29:46,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186263 2024-11-13T14:29:46,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186263 2024-11-13T14:29:46,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186263) roll requested 2024-11-13T14:29:46,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186269 2024-11-13T14:29:46,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186269 2024-11-13T14:29:46,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186269) roll requested 2024-11-13T14:29:46,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186277 2024-11-13T14:29:46,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186277 2024-11-13T14:29:46,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186277) roll requested 2024-11-13T14:29:46,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186285 2024-11-13T14:29:46,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186285 2024-11-13T14:29:46,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186285) roll requested 2024-11-13T14:29:46,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186294 2024-11-13T14:29:46,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186294 2024-11-13T14:29:46,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186294) roll requested 2024-11-13T14:29:46,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186304 2024-11-13T14:29:46,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186304 2024-11-13T14:29:46,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186304) roll requested 2024-11-13T14:29:46,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186313 2024-11-13T14:29:46,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186313 2024-11-13T14:29:46,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186313) roll requested 2024-11-13T14:29:46,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186325 2024-11-13T14:29:46,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186325 2024-11-13T14:29:46,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186325) roll requested 2024-11-13T14:29:46,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186334 2024-11-13T14:29:46,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186334 2024-11-13T14:29:46,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186334) roll requested 2024-11-13T14:29:46,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186341 2024-11-13T14:29:46,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186341 2024-11-13T14:29:46,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186341) roll requested 2024-11-13T14:29:46,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186349 2024-11-13T14:29:46,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186349 2024-11-13T14:29:46,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186349) roll requested 2024-11-13T14:29:46,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186357 2024-11-13T14:29:46,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186357 2024-11-13T14:29:46,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186357) roll requested 2024-11-13T14:29:46,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186368 2024-11-13T14:29:46,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186368 2024-11-13T14:29:46,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186368) roll requested 2024-11-13T14:29:46,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186376 2024-11-13T14:29:46,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186376 2024-11-13T14:29:46,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186376) roll requested 2024-11-13T14:29:46,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186384 2024-11-13T14:29:46,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186384 2024-11-13T14:29:46,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186384) roll requested 2024-11-13T14:29:46,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186391 2024-11-13T14:29:46,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186391 2024-11-13T14:29:46,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186391) roll requested 2024-11-13T14:29:46,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186400 2024-11-13T14:29:46,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186400 2024-11-13T14:29:46,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186400) roll requested 2024-11-13T14:29:46,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186409 2024-11-13T14:29:46,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186409 2024-11-13T14:29:46,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186409) roll requested 2024-11-13T14:29:46,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186416 2024-11-13T14:29:46,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186416 2024-11-13T14:29:46,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186416) roll requested 2024-11-13T14:29:46,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186424 2024-11-13T14:29:46,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186424 2024-11-13T14:29:46,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186424) roll requested 2024-11-13T14:29:46,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186432 2024-11-13T14:29:46,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186432 2024-11-13T14:29:46,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186432) roll requested 2024-11-13T14:29:46,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186439 2024-11-13T14:29:46,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186439 2024-11-13T14:29:46,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186439) roll requested 2024-11-13T14:29:46,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186448 2024-11-13T14:29:46,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186448 2024-11-13T14:29:46,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186448) roll requested 2024-11-13T14:29:46,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186457 2024-11-13T14:29:46,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186457 2024-11-13T14:29:46,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186457) roll requested 2024-11-13T14:29:46,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186464 2024-11-13T14:29:46,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186464 2024-11-13T14:29:46,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186464) roll requested 2024-11-13T14:29:46,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186473 2024-11-13T14:29:46,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186473 2024-11-13T14:29:46,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186473) roll requested 2024-11-13T14:29:46,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186482 2024-11-13T14:29:46,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186482 2024-11-13T14:29:46,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186482) roll requested 2024-11-13T14:29:46,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186489 2024-11-13T14:29:46,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186489 2024-11-13T14:29:46,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186489) roll requested 2024-11-13T14:29:46,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186497 2024-11-13T14:29:46,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186497 2024-11-13T14:29:46,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186497) roll requested 2024-11-13T14:29:46,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186503 2024-11-13T14:29:46,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186503 2024-11-13T14:29:46,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186503) roll requested 2024-11-13T14:29:46,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186510 2024-11-13T14:29:46,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186510 2024-11-13T14:29:46,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186510) roll requested 2024-11-13T14:29:46,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186518 2024-11-13T14:29:46,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186518 2024-11-13T14:29:46,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186518) roll requested 2024-11-13T14:29:46,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186526 2024-11-13T14:29:46,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186526 2024-11-13T14:29:46,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186526) roll requested 2024-11-13T14:29:46,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186535 2024-11-13T14:29:46,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186535 2024-11-13T14:29:46,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186535) roll requested 2024-11-13T14:29:46,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186546 2024-11-13T14:29:46,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186546 2024-11-13T14:29:46,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186546) roll requested 2024-11-13T14:29:46,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186555 2024-11-13T14:29:46,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186555 2024-11-13T14:29:46,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186555) roll requested 2024-11-13T14:29:46,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186564 2024-11-13T14:29:46,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186564 2024-11-13T14:29:46,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186564) roll requested 2024-11-13T14:29:46,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186572 2024-11-13T14:29:46,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186572 2024-11-13T14:29:46,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186572) roll requested 2024-11-13T14:29:46,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186580 2024-11-13T14:29:46,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186580 2024-11-13T14:29:46,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186580) roll requested 2024-11-13T14:29:46,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186588 2024-11-13T14:29:46,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186588 2024-11-13T14:29:46,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186588) roll requested 2024-11-13T14:29:46,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186597 2024-11-13T14:29:46,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186597 2024-11-13T14:29:46,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186597) roll requested 2024-11-13T14:29:46,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186605 2024-11-13T14:29:46,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186605 2024-11-13T14:29:46,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186605) roll requested 2024-11-13T14:29:46,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186613 2024-11-13T14:29:46,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186613 2024-11-13T14:29:46,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186613) roll requested 2024-11-13T14:29:46,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186620 2024-11-13T14:29:46,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186620 2024-11-13T14:29:46,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186620) roll requested 2024-11-13T14:29:46,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186628 2024-11-13T14:29:46,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186628 2024-11-13T14:29:46,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186628) roll requested 2024-11-13T14:29:46,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186636 2024-11-13T14:29:46,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186636 2024-11-13T14:29:46,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186636) roll requested 2024-11-13T14:29:46,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186645 2024-11-13T14:29:46,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186645 2024-11-13T14:29:46,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186645) roll requested 2024-11-13T14:29:46,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186652 2024-11-13T14:29:46,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186652 2024-11-13T14:29:46,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186652) roll requested 2024-11-13T14:29:46,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186659 2024-11-13T14:29:46,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186659 2024-11-13T14:29:46,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186659) roll requested 2024-11-13T14:29:46,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186667 2024-11-13T14:29:46,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186667 2024-11-13T14:29:46,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186667) roll requested 2024-11-13T14:29:46,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186677 2024-11-13T14:29:46,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186677 2024-11-13T14:29:46,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186677) roll requested 2024-11-13T14:29:46,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186685 2024-11-13T14:29:46,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186685 2024-11-13T14:29:46,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186685) roll requested 2024-11-13T14:29:46,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186692 2024-11-13T14:29:46,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186692 2024-11-13T14:29:46,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186692) roll requested 2024-11-13T14:29:46,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186700 2024-11-13T14:29:46,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186700 2024-11-13T14:29:46,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186700) roll requested 2024-11-13T14:29:46,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186708 2024-11-13T14:29:46,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186708 2024-11-13T14:29:46,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186708) roll requested 2024-11-13T14:29:46,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186718 2024-11-13T14:29:46,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186718 2024-11-13T14:29:46,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186718) roll requested 2024-11-13T14:29:46,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186729 2024-11-13T14:29:46,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186729 2024-11-13T14:29:46,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186729) roll requested 2024-11-13T14:29:46,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186739 2024-11-13T14:29:46,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186739 2024-11-13T14:29:46,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186739) roll requested 2024-11-13T14:29:46,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186747 2024-11-13T14:29:46,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186747 2024-11-13T14:29:46,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186747) roll requested 2024-11-13T14:29:46,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186754 2024-11-13T14:29:46,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186754 2024-11-13T14:29:46,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186754) roll requested 2024-11-13T14:29:46,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186763 2024-11-13T14:29:46,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186763 2024-11-13T14:29:46,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186763) roll requested 2024-11-13T14:29:46,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186771 2024-11-13T14:29:46,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186771 2024-11-13T14:29:46,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186771) roll requested 2024-11-13T14:29:46,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186777 2024-11-13T14:29:46,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186777 2024-11-13T14:29:46,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186777) roll requested 2024-11-13T14:29:46,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186786 2024-11-13T14:29:46,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186786 2024-11-13T14:29:46,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186786) roll requested 2024-11-13T14:29:46,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186795 2024-11-13T14:29:46,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186795 2024-11-13T14:29:46,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186795) roll requested 2024-11-13T14:29:46,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186804 2024-11-13T14:29:46,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186804 2024-11-13T14:29:46,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186804) roll requested 2024-11-13T14:29:46,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186812 2024-11-13T14:29:46,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186812 2024-11-13T14:29:46,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186812) roll requested 2024-11-13T14:29:46,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186821 2024-11-13T14:29:46,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186821 2024-11-13T14:29:46,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186821) roll requested 2024-11-13T14:29:46,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186830 2024-11-13T14:29:46,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186830 2024-11-13T14:29:46,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186830) roll requested 2024-11-13T14:29:46,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186838 2024-11-13T14:29:46,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186838 2024-11-13T14:29:46,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186838) roll requested 2024-11-13T14:29:46,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186848 2024-11-13T14:29:46,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186848 2024-11-13T14:29:46,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186848) roll requested 2024-11-13T14:29:46,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186858 2024-11-13T14:29:46,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186858 2024-11-13T14:29:46,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186858) roll requested 2024-11-13T14:29:46,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186868 2024-11-13T14:29:46,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186868 2024-11-13T14:29:46,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186868) roll requested 2024-11-13T14:29:46,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186877 2024-11-13T14:29:46,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186877 2024-11-13T14:29:46,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186877) roll requested 2024-11-13T14:29:46,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186886 2024-11-13T14:29:46,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186886 2024-11-13T14:29:46,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186886) roll requested 2024-11-13T14:29:46,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186893 2024-11-13T14:29:46,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186893 2024-11-13T14:29:46,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186893) roll requested 2024-11-13T14:29:46,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186903 2024-11-13T14:29:46,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186903 2024-11-13T14:29:46,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186903) roll requested 2024-11-13T14:29:46,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186910 2024-11-13T14:29:46,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186910 2024-11-13T14:29:46,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186910) roll requested 2024-11-13T14:29:46,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186917 2024-11-13T14:29:46,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186917 2024-11-13T14:29:46,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186917) roll requested 2024-11-13T14:29:46,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186926 2024-11-13T14:29:46,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186926 2024-11-13T14:29:46,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186926) roll requested 2024-11-13T14:29:46,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186936 2024-11-13T14:29:46,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186936 2024-11-13T14:29:46,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186936) roll requested 2024-11-13T14:29:46,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186949 2024-11-13T14:29:46,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186949 2024-11-13T14:29:46,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186949) roll requested 2024-11-13T14:29:46,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186957 2024-11-13T14:29:46,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186957 2024-11-13T14:29:46,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186957) roll requested 2024-11-13T14:29:46,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186963 2024-11-13T14:29:46,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186963 2024-11-13T14:29:46,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186963) roll requested 2024-11-13T14:29:46,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186969 2024-11-13T14:29:46,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186969 2024-11-13T14:29:46,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186969) roll requested 2024-11-13T14:29:46,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186975 2024-11-13T14:29:46,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186975 2024-11-13T14:29:46,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186975) roll requested 2024-11-13T14:29:46,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186983 2024-11-13T14:29:46,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186983 2024-11-13T14:29:46,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186983) roll requested 2024-11-13T14:29:46,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186991 2024-11-13T14:29:46,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:46,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186991 2024-11-13T14:29:46,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:46,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:46,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:46,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:46,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186991) roll requested 2024-11-13T14:29:46,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508186999 2024-11-13T14:29:47,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186999 2024-11-13T14:29:47,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508186999) roll requested 2024-11-13T14:29:47,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187009 2024-11-13T14:29:47,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508186999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187009 2024-11-13T14:29:47,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187009) roll requested 2024-11-13T14:29:47,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187018 2024-11-13T14:29:47,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187018 2024-11-13T14:29:47,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187018) roll requested 2024-11-13T14:29:47,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187028 2024-11-13T14:29:47,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187028 2024-11-13T14:29:47,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187028) roll requested 2024-11-13T14:29:47,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187037 2024-11-13T14:29:47,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187037 2024-11-13T14:29:47,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187037) roll requested 2024-11-13T14:29:47,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187043 2024-11-13T14:29:47,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187043 2024-11-13T14:29:47,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187043) roll requested 2024-11-13T14:29:47,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187049 2024-11-13T14:29:47,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187049 2024-11-13T14:29:47,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187049) roll requested 2024-11-13T14:29:47,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187057 2024-11-13T14:29:47,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187057 2024-11-13T14:29:47,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187057) roll requested 2024-11-13T14:29:47,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187066 2024-11-13T14:29:47,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187066 2024-11-13T14:29:47,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187066) roll requested 2024-11-13T14:29:47,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187075 2024-11-13T14:29:47,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187075 2024-11-13T14:29:47,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187075) roll requested 2024-11-13T14:29:47,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187085 2024-11-13T14:29:47,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187085 2024-11-13T14:29:47,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187085) roll requested 2024-11-13T14:29:47,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187092 2024-11-13T14:29:47,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187092 2024-11-13T14:29:47,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187092) roll requested 2024-11-13T14:29:47,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187100 2024-11-13T14:29:47,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187100 2024-11-13T14:29:47,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187100) roll requested 2024-11-13T14:29:47,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187108 2024-11-13T14:29:47,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187108 2024-11-13T14:29:47,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187108) roll requested 2024-11-13T14:29:47,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187117 2024-11-13T14:29:47,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187117 2024-11-13T14:29:47,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187117) roll requested 2024-11-13T14:29:47,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187125 2024-11-13T14:29:47,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187125 2024-11-13T14:29:47,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187125) roll requested 2024-11-13T14:29:47,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187135 2024-11-13T14:29:47,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187135 2024-11-13T14:29:47,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187135) roll requested 2024-11-13T14:29:47,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187145 2024-11-13T14:29:47,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187145 2024-11-13T14:29:47,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187145) roll requested 2024-11-13T14:29:47,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187153 2024-11-13T14:29:47,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187153 2024-11-13T14:29:47,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187153) roll requested 2024-11-13T14:29:47,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187162 2024-11-13T14:29:47,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187162 2024-11-13T14:29:47,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187162) roll requested 2024-11-13T14:29:47,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187170 2024-11-13T14:29:47,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187170 2024-11-13T14:29:47,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187170) roll requested 2024-11-13T14:29:47,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187177 2024-11-13T14:29:47,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187177 2024-11-13T14:29:47,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187177) roll requested 2024-11-13T14:29:47,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187186 2024-11-13T14:29:47,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187186 2024-11-13T14:29:47,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187186) roll requested 2024-11-13T14:29:47,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187195 2024-11-13T14:29:47,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187195 2024-11-13T14:29:47,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187195) roll requested 2024-11-13T14:29:47,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187204 2024-11-13T14:29:47,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187204 2024-11-13T14:29:47,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187204) roll requested 2024-11-13T14:29:47,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187212 2024-11-13T14:29:47,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187212 2024-11-13T14:29:47,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187212) roll requested 2024-11-13T14:29:47,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187223 2024-11-13T14:29:47,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187223 2024-11-13T14:29:47,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187223) roll requested 2024-11-13T14:29:47,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187233 2024-11-13T14:29:47,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187233 2024-11-13T14:29:47,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187233) roll requested 2024-11-13T14:29:47,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187241 2024-11-13T14:29:47,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187241 2024-11-13T14:29:47,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187241) roll requested 2024-11-13T14:29:47,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187246 2024-11-13T14:29:47,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187246 2024-11-13T14:29:47,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187246) roll requested 2024-11-13T14:29:47,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187251 2024-11-13T14:29:47,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187251 2024-11-13T14:29:47,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187251) roll requested 2024-11-13T14:29:47,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187256 2024-11-13T14:29:47,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187256 2024-11-13T14:29:47,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187256) roll requested 2024-11-13T14:29:47,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187263 2024-11-13T14:29:47,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187263 2024-11-13T14:29:47,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187263) roll requested 2024-11-13T14:29:47,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187270 2024-11-13T14:29:47,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187270 2024-11-13T14:29:47,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187270) roll requested 2024-11-13T14:29:47,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187278 2024-11-13T14:29:47,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187278 2024-11-13T14:29:47,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187278) roll requested 2024-11-13T14:29:47,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187287 2024-11-13T14:29:47,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187287 2024-11-13T14:29:47,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187287) roll requested 2024-11-13T14:29:47,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187297 2024-11-13T14:29:47,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187297 2024-11-13T14:29:47,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187297) roll requested 2024-11-13T14:29:47,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187307 2024-11-13T14:29:47,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187307 2024-11-13T14:29:47,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187307) roll requested 2024-11-13T14:29:47,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187314 2024-11-13T14:29:47,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187314 2024-11-13T14:29:47,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187314) roll requested 2024-11-13T14:29:47,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187321 2024-11-13T14:29:47,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187321 2024-11-13T14:29:47,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187321) roll requested 2024-11-13T14:29:47,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187329 2024-11-13T14:29:47,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187329 2024-11-13T14:29:47,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187329) roll requested 2024-11-13T14:29:47,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187339 2024-11-13T14:29:47,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187339 2024-11-13T14:29:47,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187339) roll requested 2024-11-13T14:29:47,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187349 2024-11-13T14:29:47,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187349 2024-11-13T14:29:47,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187349) roll requested 2024-11-13T14:29:47,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187361 2024-11-13T14:29:47,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187361 2024-11-13T14:29:47,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187361) roll requested 2024-11-13T14:29:47,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187367 2024-11-13T14:29:47,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187367 2024-11-13T14:29:47,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187367) roll requested 2024-11-13T14:29:47,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187373 2024-11-13T14:29:47,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187373 2024-11-13T14:29:47,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187373) roll requested 2024-11-13T14:29:47,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187380 2024-11-13T14:29:47,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187380 2024-11-13T14:29:47,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187380) roll requested 2024-11-13T14:29:47,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187386 2024-11-13T14:29:47,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187386 2024-11-13T14:29:47,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187386) roll requested 2024-11-13T14:29:47,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187393 2024-11-13T14:29:47,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187393 2024-11-13T14:29:47,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187393) roll requested 2024-11-13T14:29:47,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187399 2024-11-13T14:29:47,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187399 2024-11-13T14:29:47,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187399) roll requested 2024-11-13T14:29:47,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187406 2024-11-13T14:29:47,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187406 2024-11-13T14:29:47,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187406) roll requested 2024-11-13T14:29:47,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187411 2024-11-13T14:29:47,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187411 2024-11-13T14:29:47,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187411) roll requested 2024-11-13T14:29:47,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187421 2024-11-13T14:29:47,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187421 2024-11-13T14:29:47,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187421) roll requested 2024-11-13T14:29:47,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187429 2024-11-13T14:29:47,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187429 2024-11-13T14:29:47,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187429) roll requested 2024-11-13T14:29:47,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187434 2024-11-13T14:29:47,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187434 2024-11-13T14:29:47,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187434) roll requested 2024-11-13T14:29:47,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187440 2024-11-13T14:29:47,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187440 2024-11-13T14:29:47,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187440) roll requested 2024-11-13T14:29:47,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187449 2024-11-13T14:29:47,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187449 2024-11-13T14:29:47,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187449) roll requested 2024-11-13T14:29:47,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187454 2024-11-13T14:29:47,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187454 2024-11-13T14:29:47,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187454) roll requested 2024-11-13T14:29:47,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187460 2024-11-13T14:29:47,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187460 2024-11-13T14:29:47,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187460) roll requested 2024-11-13T14:29:47,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187466 2024-11-13T14:29:47,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187466 2024-11-13T14:29:47,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187466) roll requested 2024-11-13T14:29:47,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187471 2024-11-13T14:29:47,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187471 2024-11-13T14:29:47,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187471) roll requested 2024-11-13T14:29:47,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187477 2024-11-13T14:29:47,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187477 2024-11-13T14:29:47,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187477) roll requested 2024-11-13T14:29:47,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187484 2024-11-13T14:29:47,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187484 2024-11-13T14:29:47,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187484) roll requested 2024-11-13T14:29:47,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187490 2024-11-13T14:29:47,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187490 2024-11-13T14:29:47,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187490) roll requested 2024-11-13T14:29:47,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187495 2024-11-13T14:29:47,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187495 2024-11-13T14:29:47,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187495) roll requested 2024-11-13T14:29:47,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187501 2024-11-13T14:29:47,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187501 2024-11-13T14:29:47,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187501) roll requested 2024-11-13T14:29:47,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187506 2024-11-13T14:29:47,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187506 2024-11-13T14:29:47,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187506) roll requested 2024-11-13T14:29:47,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187511 2024-11-13T14:29:47,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187511 2024-11-13T14:29:47,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187511) roll requested 2024-11-13T14:29:47,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187519 2024-11-13T14:29:47,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187519 2024-11-13T14:29:47,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187519) roll requested 2024-11-13T14:29:47,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187527 2024-11-13T14:29:47,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187527 2024-11-13T14:29:47,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187527) roll requested 2024-11-13T14:29:47,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187536 2024-11-13T14:29:47,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187536 2024-11-13T14:29:47,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187536) roll requested 2024-11-13T14:29:47,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187545 2024-11-13T14:29:47,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187545 2024-11-13T14:29:47,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187545) roll requested 2024-11-13T14:29:47,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187554 2024-11-13T14:29:47,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187554 2024-11-13T14:29:47,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187554) roll requested 2024-11-13T14:29:47,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187563 2024-11-13T14:29:47,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187563 2024-11-13T14:29:47,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187563) roll requested 2024-11-13T14:29:47,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187574 2024-11-13T14:29:47,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187574 2024-11-13T14:29:47,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187574) roll requested 2024-11-13T14:29:47,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187584 2024-11-13T14:29:47,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187584 2024-11-13T14:29:47,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187584) roll requested 2024-11-13T14:29:47,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187595 2024-11-13T14:29:47,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187595 2024-11-13T14:29:47,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187595) roll requested 2024-11-13T14:29:47,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187604 2024-11-13T14:29:47,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187604 2024-11-13T14:29:47,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187604) roll requested 2024-11-13T14:29:47,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187612 2024-11-13T14:29:47,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187612 2024-11-13T14:29:47,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187612) roll requested 2024-11-13T14:29:47,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187621 2024-11-13T14:29:47,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187621 2024-11-13T14:29:47,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187621) roll requested 2024-11-13T14:29:47,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187629 2024-11-13T14:29:47,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187629 2024-11-13T14:29:47,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187629) roll requested 2024-11-13T14:29:47,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187637 2024-11-13T14:29:47,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187637 2024-11-13T14:29:47,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187637) roll requested 2024-11-13T14:29:47,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187645 2024-11-13T14:29:47,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187645 2024-11-13T14:29:47,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187645) roll requested 2024-11-13T14:29:47,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187653 2024-11-13T14:29:47,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187653 2024-11-13T14:29:47,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187653) roll requested 2024-11-13T14:29:47,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187661 2024-11-13T14:29:47,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187661 2024-11-13T14:29:47,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187661) roll requested 2024-11-13T14:29:47,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187669 2024-11-13T14:29:47,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187669 2024-11-13T14:29:47,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187669) roll requested 2024-11-13T14:29:47,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187678 2024-11-13T14:29:47,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187678 2024-11-13T14:29:47,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187678) roll requested 2024-11-13T14:29:47,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187687 2024-11-13T14:29:47,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187687 2024-11-13T14:29:47,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187687) roll requested 2024-11-13T14:29:47,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187696 2024-11-13T14:29:47,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187696 2024-11-13T14:29:47,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187696) roll requested 2024-11-13T14:29:47,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187703 2024-11-13T14:29:47,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187703 2024-11-13T14:29:47,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187703) roll requested 2024-11-13T14:29:47,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187710 2024-11-13T14:29:47,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187710 2024-11-13T14:29:47,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187710) roll requested 2024-11-13T14:29:47,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187716 2024-11-13T14:29:47,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187716 2024-11-13T14:29:47,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187716) roll requested 2024-11-13T14:29:47,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187723 2024-11-13T14:29:47,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187723 2024-11-13T14:29:47,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187723) roll requested 2024-11-13T14:29:47,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187730 2024-11-13T14:29:47,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187730 2024-11-13T14:29:47,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187730) roll requested 2024-11-13T14:29:47,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187740 2024-11-13T14:29:47,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187740 2024-11-13T14:29:47,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187740) roll requested 2024-11-13T14:29:47,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187748 2024-11-13T14:29:47,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187748 2024-11-13T14:29:47,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187748) roll requested 2024-11-13T14:29:47,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187758 2024-11-13T14:29:47,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187758 2024-11-13T14:29:47,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187758) roll requested 2024-11-13T14:29:47,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187769 2024-11-13T14:29:47,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187769 2024-11-13T14:29:47,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187769) roll requested 2024-11-13T14:29:47,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187777 2024-11-13T14:29:47,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187777 2024-11-13T14:29:47,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187777) roll requested 2024-11-13T14:29:47,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187782 2024-11-13T14:29:47,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187782 2024-11-13T14:29:47,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187782) roll requested 2024-11-13T14:29:47,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187788 2024-11-13T14:29:47,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187788 2024-11-13T14:29:47,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187788) roll requested 2024-11-13T14:29:47,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187797 2024-11-13T14:29:47,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187797 2024-11-13T14:29:47,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187797) roll requested 2024-11-13T14:29:47,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187803 2024-11-13T14:29:47,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187803 2024-11-13T14:29:47,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187803) roll requested 2024-11-13T14:29:47,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187808 2024-11-13T14:29:47,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187808 2024-11-13T14:29:47,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187808) roll requested 2024-11-13T14:29:47,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187813 2024-11-13T14:29:47,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187813 2024-11-13T14:29:47,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187813) roll requested 2024-11-13T14:29:47,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187820 2024-11-13T14:29:47,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187820 2024-11-13T14:29:47,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187820) roll requested 2024-11-13T14:29:47,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187828 2024-11-13T14:29:47,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187828 2024-11-13T14:29:47,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187828) roll requested 2024-11-13T14:29:47,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187837 2024-11-13T14:29:47,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187837 2024-11-13T14:29:47,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187837) roll requested 2024-11-13T14:29:47,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187845 2024-11-13T14:29:47,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187845 2024-11-13T14:29:47,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187845) roll requested 2024-11-13T14:29:47,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187850 2024-11-13T14:29:47,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187850 2024-11-13T14:29:47,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187850) roll requested 2024-11-13T14:29:47,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187856 2024-11-13T14:29:47,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187856 2024-11-13T14:29:47,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187856) roll requested 2024-11-13T14:29:47,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187862 2024-11-13T14:29:47,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187862 2024-11-13T14:29:47,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187862) roll requested 2024-11-13T14:29:47,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187868 2024-11-13T14:29:47,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187868 2024-11-13T14:29:47,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187868) roll requested 2024-11-13T14:29:47,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187874 2024-11-13T14:29:47,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187874 2024-11-13T14:29:47,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187874) roll requested 2024-11-13T14:29:47,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187881 2024-11-13T14:29:47,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187881 2024-11-13T14:29:47,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187881) roll requested 2024-11-13T14:29:47,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187888 2024-11-13T14:29:47,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187888 2024-11-13T14:29:47,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187888) roll requested 2024-11-13T14:29:47,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187893 2024-11-13T14:29:47,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187893 2024-11-13T14:29:47,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187893) roll requested 2024-11-13T14:29:47,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187900 2024-11-13T14:29:47,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187900 2024-11-13T14:29:47,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187900) roll requested 2024-11-13T14:29:47,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187906 2024-11-13T14:29:47,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187906 2024-11-13T14:29:47,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187906) roll requested 2024-11-13T14:29:47,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187912 2024-11-13T14:29:47,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187912 2024-11-13T14:29:47,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187912) roll requested 2024-11-13T14:29:47,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187917 2024-11-13T14:29:47,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187917 2024-11-13T14:29:47,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187917) roll requested 2024-11-13T14:29:47,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187923 2024-11-13T14:29:47,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187923 2024-11-13T14:29:47,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187923) roll requested 2024-11-13T14:29:47,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187930 2024-11-13T14:29:47,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187930 2024-11-13T14:29:47,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187930) roll requested 2024-11-13T14:29:47,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187939 2024-11-13T14:29:47,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187939 2024-11-13T14:29:47,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187939) roll requested 2024-11-13T14:29:47,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187945 2024-11-13T14:29:47,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187945 2024-11-13T14:29:47,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187945) roll requested 2024-11-13T14:29:47,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187951 2024-11-13T14:29:47,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187951 2024-11-13T14:29:47,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187951) roll requested 2024-11-13T14:29:47,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187956 2024-11-13T14:29:47,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187956 2024-11-13T14:29:47,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187956) roll requested 2024-11-13T14:29:47,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187962 2024-11-13T14:29:47,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187962 2024-11-13T14:29:47,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187962) roll requested 2024-11-13T14:29:47,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187969 2024-11-13T14:29:47,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187969 2024-11-13T14:29:47,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187969) roll requested 2024-11-13T14:29:47,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187975 2024-11-13T14:29:47,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187975 2024-11-13T14:29:47,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187975) roll requested 2024-11-13T14:29:47,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187980 2024-11-13T14:29:47,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187980 2024-11-13T14:29:47,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187980) roll requested 2024-11-13T14:29:47,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187987 2024-11-13T14:29:47,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187987 2024-11-13T14:29:47,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:47,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:47,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187987) roll requested 2024-11-13T14:29:47,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508187993 2024-11-13T14:29:47,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:47,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187993 2024-11-13T14:29:47,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:47,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:47,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508187993) roll requested 2024-11-13T14:29:48,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188000 2024-11-13T14:29:48,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508187993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188000 2024-11-13T14:29:48,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188000) roll requested 2024-11-13T14:29:48,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188006 2024-11-13T14:29:48,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188006 2024-11-13T14:29:48,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188006) roll requested 2024-11-13T14:29:48,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188012 2024-11-13T14:29:48,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188012 2024-11-13T14:29:48,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188012) roll requested 2024-11-13T14:29:48,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188019 2024-11-13T14:29:48,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188019 2024-11-13T14:29:48,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188019) roll requested 2024-11-13T14:29:48,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188024 2024-11-13T14:29:48,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188024 2024-11-13T14:29:48,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188024) roll requested 2024-11-13T14:29:48,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188030 2024-11-13T14:29:48,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188030 2024-11-13T14:29:48,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188030) roll requested 2024-11-13T14:29:48,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188037 2024-11-13T14:29:48,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188037 2024-11-13T14:29:48,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188037) roll requested 2024-11-13T14:29:48,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188042 2024-11-13T14:29:48,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188042 2024-11-13T14:29:48,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188042) roll requested 2024-11-13T14:29:48,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188050 2024-11-13T14:29:48,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188050 2024-11-13T14:29:48,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188050) roll requested 2024-11-13T14:29:48,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188057 2024-11-13T14:29:48,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188057 2024-11-13T14:29:48,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188057) roll requested 2024-11-13T14:29:48,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188063 2024-11-13T14:29:48,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188063 2024-11-13T14:29:48,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188063) roll requested 2024-11-13T14:29:48,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188069 2024-11-13T14:29:48,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188069 2024-11-13T14:29:48,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188069) roll requested 2024-11-13T14:29:48,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188075 2024-11-13T14:29:48,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188075 2024-11-13T14:29:48,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188075) roll requested 2024-11-13T14:29:48,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188081 2024-11-13T14:29:48,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188081 2024-11-13T14:29:48,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188081) roll requested 2024-11-13T14:29:48,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188090 2024-11-13T14:29:48,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188090 2024-11-13T14:29:48,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188090) roll requested 2024-11-13T14:29:48,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188096 2024-11-13T14:29:48,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188096 2024-11-13T14:29:48,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188096) roll requested 2024-11-13T14:29:48,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188103 2024-11-13T14:29:48,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188103 2024-11-13T14:29:48,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188103) roll requested 2024-11-13T14:29:48,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188109 2024-11-13T14:29:48,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188109 2024-11-13T14:29:48,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188109) roll requested 2024-11-13T14:29:48,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188115 2024-11-13T14:29:48,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188115 2024-11-13T14:29:48,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188115) roll requested 2024-11-13T14:29:48,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188121 2024-11-13T14:29:48,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188121 2024-11-13T14:29:48,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188121) roll requested 2024-11-13T14:29:48,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188127 2024-11-13T14:29:48,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188127 2024-11-13T14:29:48,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188127) roll requested 2024-11-13T14:29:48,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188132 2024-11-13T14:29:48,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188132 2024-11-13T14:29:48,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188132) roll requested 2024-11-13T14:29:48,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188138 2024-11-13T14:29:48,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188138 2024-11-13T14:29:48,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188138) roll requested 2024-11-13T14:29:48,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188144 2024-11-13T14:29:48,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188144 2024-11-13T14:29:48,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188144) roll requested 2024-11-13T14:29:48,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188149 2024-11-13T14:29:48,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188149 2024-11-13T14:29:48,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188149) roll requested 2024-11-13T14:29:48,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188154 2024-11-13T14:29:48,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188154 2024-11-13T14:29:48,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188154) roll requested 2024-11-13T14:29:48,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188161 2024-11-13T14:29:48,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188161 2024-11-13T14:29:48,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188161) roll requested 2024-11-13T14:29:48,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188168 2024-11-13T14:29:48,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188168 2024-11-13T14:29:48,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188168) roll requested 2024-11-13T14:29:48,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188176 2024-11-13T14:29:48,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188176 2024-11-13T14:29:48,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188176) roll requested 2024-11-13T14:29:48,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188185 2024-11-13T14:29:48,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188185 2024-11-13T14:29:48,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188185) roll requested 2024-11-13T14:29:48,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188193 2024-11-13T14:29:48,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188193 2024-11-13T14:29:48,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188193) roll requested 2024-11-13T14:29:48,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188201 2024-11-13T14:29:48,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188201 2024-11-13T14:29:48,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188201) roll requested 2024-11-13T14:29:48,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188209 2024-11-13T14:29:48,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188209 2024-11-13T14:29:48,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188209) roll requested 2024-11-13T14:29:48,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188219 2024-11-13T14:29:48,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188219 2024-11-13T14:29:48,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188219) roll requested 2024-11-13T14:29:48,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188229 2024-11-13T14:29:48,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188229 2024-11-13T14:29:48,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188229) roll requested 2024-11-13T14:29:48,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188239 2024-11-13T14:29:48,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188239 2024-11-13T14:29:48,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188239) roll requested 2024-11-13T14:29:48,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188248 2024-11-13T14:29:48,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188248 2024-11-13T14:29:48,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188248) roll requested 2024-11-13T14:29:48,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188256 2024-11-13T14:29:48,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188256 2024-11-13T14:29:48,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188256) roll requested 2024-11-13T14:29:48,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188265 2024-11-13T14:29:48,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188265 2024-11-13T14:29:48,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188265) roll requested 2024-11-13T14:29:48,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188273 2024-11-13T14:29:48,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188273 2024-11-13T14:29:48,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188273) roll requested 2024-11-13T14:29:48,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188279 2024-11-13T14:29:48,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188279 2024-11-13T14:29:48,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188279) roll requested 2024-11-13T14:29:48,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188286 2024-11-13T14:29:48,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188286 2024-11-13T14:29:48,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188286) roll requested 2024-11-13T14:29:48,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188291 2024-11-13T14:29:48,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188291 2024-11-13T14:29:48,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188291) roll requested 2024-11-13T14:29:48,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188302 2024-11-13T14:29:48,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188302 2024-11-13T14:29:48,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188302) roll requested 2024-11-13T14:29:48,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188308 2024-11-13T14:29:48,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188308 2024-11-13T14:29:48,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188308) roll requested 2024-11-13T14:29:48,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188318 2024-11-13T14:29:48,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188318 2024-11-13T14:29:48,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188318) roll requested 2024-11-13T14:29:48,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188325 2024-11-13T14:29:48,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188325 2024-11-13T14:29:48,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188325) roll requested 2024-11-13T14:29:48,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188331 2024-11-13T14:29:48,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188331 2024-11-13T14:29:48,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188331) roll requested 2024-11-13T14:29:48,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188337 2024-11-13T14:29:48,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188337 2024-11-13T14:29:48,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188337) roll requested 2024-11-13T14:29:48,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188343 2024-11-13T14:29:48,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188343 2024-11-13T14:29:48,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188343) roll requested 2024-11-13T14:29:48,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188348 2024-11-13T14:29:48,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188348 2024-11-13T14:29:48,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188348) roll requested 2024-11-13T14:29:48,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188354 2024-11-13T14:29:48,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188354 2024-11-13T14:29:48,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188354) roll requested 2024-11-13T14:29:48,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188360 2024-11-13T14:29:48,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188360 2024-11-13T14:29:48,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188360) roll requested 2024-11-13T14:29:48,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188371 2024-11-13T14:29:48,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188371 2024-11-13T14:29:48,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188371) roll requested 2024-11-13T14:29:48,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188380 2024-11-13T14:29:48,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188380 2024-11-13T14:29:48,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188380) roll requested 2024-11-13T14:29:48,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188386 2024-11-13T14:29:48,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188386 2024-11-13T14:29:48,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188386) roll requested 2024-11-13T14:29:48,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188392 2024-11-13T14:29:48,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188392 2024-11-13T14:29:48,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188392) roll requested 2024-11-13T14:29:48,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188397 2024-11-13T14:29:48,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188397 2024-11-13T14:29:48,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188397) roll requested 2024-11-13T14:29:48,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188402 2024-11-13T14:29:48,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188402 2024-11-13T14:29:48,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188402) roll requested 2024-11-13T14:29:48,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188408 2024-11-13T14:29:48,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188408 2024-11-13T14:29:48,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188408) roll requested 2024-11-13T14:29:48,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188414 2024-11-13T14:29:48,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188414 2024-11-13T14:29:48,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188414) roll requested 2024-11-13T14:29:48,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188421 2024-11-13T14:29:48,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188421 2024-11-13T14:29:48,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188421) roll requested 2024-11-13T14:29:48,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188429 2024-11-13T14:29:48,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188429 2024-11-13T14:29:48,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188429) roll requested 2024-11-13T14:29:48,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188435 2024-11-13T14:29:48,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188435 2024-11-13T14:29:48,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188435) roll requested 2024-11-13T14:29:48,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188442 2024-11-13T14:29:48,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188442 2024-11-13T14:29:48,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188442) roll requested 2024-11-13T14:29:48,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188450 2024-11-13T14:29:48,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188450 2024-11-13T14:29:48,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188450) roll requested 2024-11-13T14:29:48,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188459 2024-11-13T14:29:48,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188459 2024-11-13T14:29:48,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188459) roll requested 2024-11-13T14:29:48,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188467 2024-11-13T14:29:48,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188467 2024-11-13T14:29:48,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188467) roll requested 2024-11-13T14:29:48,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188474 2024-11-13T14:29:48,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188474 2024-11-13T14:29:48,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188474) roll requested 2024-11-13T14:29:48,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188482 2024-11-13T14:29:48,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188482 2024-11-13T14:29:48,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188482) roll requested 2024-11-13T14:29:48,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188492 2024-11-13T14:29:48,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188492 2024-11-13T14:29:48,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188492) roll requested 2024-11-13T14:29:48,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188500 2024-11-13T14:29:48,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188500 2024-11-13T14:29:48,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188500) roll requested 2024-11-13T14:29:48,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188508 2024-11-13T14:29:48,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188508 2024-11-13T14:29:48,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188508) roll requested 2024-11-13T14:29:48,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188516 2024-11-13T14:29:48,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188516 2024-11-13T14:29:48,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188516) roll requested 2024-11-13T14:29:48,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188525 2024-11-13T14:29:48,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188525 2024-11-13T14:29:48,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188525) roll requested 2024-11-13T14:29:48,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188533 2024-11-13T14:29:48,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188533 2024-11-13T14:29:48,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188533) roll requested 2024-11-13T14:29:48,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188542 2024-11-13T14:29:48,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188542 2024-11-13T14:29:48,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188542) roll requested 2024-11-13T14:29:48,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188550 2024-11-13T14:29:48,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188550 2024-11-13T14:29:48,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188550) roll requested 2024-11-13T14:29:48,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188556 2024-11-13T14:29:48,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188556 2024-11-13T14:29:48,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188556) roll requested 2024-11-13T14:29:48,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188564 2024-11-13T14:29:48,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188564 2024-11-13T14:29:48,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188564) roll requested 2024-11-13T14:29:48,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188572 2024-11-13T14:29:48,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188572 2024-11-13T14:29:48,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188572) roll requested 2024-11-13T14:29:48,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188582 2024-11-13T14:29:48,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188582 2024-11-13T14:29:48,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188582) roll requested 2024-11-13T14:29:48,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188591 2024-11-13T14:29:48,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188591 2024-11-13T14:29:48,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188591) roll requested 2024-11-13T14:29:48,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188598 2024-11-13T14:29:48,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188598 2024-11-13T14:29:48,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188598) roll requested 2024-11-13T14:29:48,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188604 2024-11-13T14:29:48,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188604 2024-11-13T14:29:48,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188604) roll requested 2024-11-13T14:29:48,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188611 2024-11-13T14:29:48,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188611 2024-11-13T14:29:48,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188611) roll requested 2024-11-13T14:29:48,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188618 2024-11-13T14:29:48,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188618 2024-11-13T14:29:48,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188618) roll requested 2024-11-13T14:29:48,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188627 2024-11-13T14:29:48,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188627 2024-11-13T14:29:48,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188627) roll requested 2024-11-13T14:29:48,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188636 2024-11-13T14:29:48,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188636 2024-11-13T14:29:48,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188636) roll requested 2024-11-13T14:29:48,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188645 2024-11-13T14:29:48,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188645 2024-11-13T14:29:48,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188645) roll requested 2024-11-13T14:29:48,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188655 2024-11-13T14:29:48,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188655 2024-11-13T14:29:48,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188655) roll requested 2024-11-13T14:29:48,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188663 2024-11-13T14:29:48,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188663 2024-11-13T14:29:48,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188663) roll requested 2024-11-13T14:29:48,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188672 2024-11-13T14:29:48,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188672 2024-11-13T14:29:48,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188672) roll requested 2024-11-13T14:29:48,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188678 2024-11-13T14:29:48,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188678 2024-11-13T14:29:48,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188678) roll requested 2024-11-13T14:29:48,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188686 2024-11-13T14:29:48,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188686 2024-11-13T14:29:48,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188686) roll requested 2024-11-13T14:29:48,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188693 2024-11-13T14:29:48,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188693 2024-11-13T14:29:48,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188693) roll requested 2024-11-13T14:29:48,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188701 2024-11-13T14:29:48,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188701 2024-11-13T14:29:48,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188701) roll requested 2024-11-13T14:29:48,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188708 2024-11-13T14:29:48,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188708 2024-11-13T14:29:48,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188708) roll requested 2024-11-13T14:29:48,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188716 2024-11-13T14:29:48,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188716 2024-11-13T14:29:48,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188716) roll requested 2024-11-13T14:29:48,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188724 2024-11-13T14:29:48,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188724 2024-11-13T14:29:48,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188724) roll requested 2024-11-13T14:29:48,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188733 2024-11-13T14:29:48,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188733 2024-11-13T14:29:48,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188733) roll requested 2024-11-13T14:29:48,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188743 2024-11-13T14:29:48,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188743 2024-11-13T14:29:48,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188743) roll requested 2024-11-13T14:29:48,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188752 2024-11-13T14:29:48,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188752 2024-11-13T14:29:48,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188752) roll requested 2024-11-13T14:29:48,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188761 2024-11-13T14:29:48,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188761 2024-11-13T14:29:48,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188761) roll requested 2024-11-13T14:29:48,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188769 2024-11-13T14:29:48,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188769 2024-11-13T14:29:48,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188769) roll requested 2024-11-13T14:29:48,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188776 2024-11-13T14:29:48,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188776 2024-11-13T14:29:48,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188776) roll requested 2024-11-13T14:29:48,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188785 2024-11-13T14:29:48,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188785 2024-11-13T14:29:48,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188785) roll requested 2024-11-13T14:29:48,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188795 2024-11-13T14:29:48,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188795 2024-11-13T14:29:48,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188795) roll requested 2024-11-13T14:29:48,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188804 2024-11-13T14:29:48,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188804 2024-11-13T14:29:48,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188804) roll requested 2024-11-13T14:29:48,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188813 2024-11-13T14:29:48,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188813 2024-11-13T14:29:48,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188813) roll requested 2024-11-13T14:29:48,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188822 2024-11-13T14:29:48,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188822 2024-11-13T14:29:48,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188822) roll requested 2024-11-13T14:29:48,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188831 2024-11-13T14:29:48,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188831 2024-11-13T14:29:48,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188831) roll requested 2024-11-13T14:29:48,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188839 2024-11-13T14:29:48,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188839 2024-11-13T14:29:48,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188839) roll requested 2024-11-13T14:29:48,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188847 2024-11-13T14:29:48,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188847 2024-11-13T14:29:48,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188847) roll requested 2024-11-13T14:29:48,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188855 2024-11-13T14:29:48,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188855 2024-11-13T14:29:48,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188855) roll requested 2024-11-13T14:29:48,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188865 2024-11-13T14:29:48,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188865 2024-11-13T14:29:48,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188865) roll requested 2024-11-13T14:29:48,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188874 2024-11-13T14:29:48,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188874 2024-11-13T14:29:48,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188874) roll requested 2024-11-13T14:29:48,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188883 2024-11-13T14:29:48,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188883 2024-11-13T14:29:48,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188883) roll requested 2024-11-13T14:29:48,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188893 2024-11-13T14:29:48,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188893 2024-11-13T14:29:48,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188893) roll requested 2024-11-13T14:29:48,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188901 2024-11-13T14:29:48,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188901 2024-11-13T14:29:48,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188901) roll requested 2024-11-13T14:29:48,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188908 2024-11-13T14:29:48,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188908 2024-11-13T14:29:48,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188908) roll requested 2024-11-13T14:29:48,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188916 2024-11-13T14:29:48,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188916 2024-11-13T14:29:48,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188916) roll requested 2024-11-13T14:29:48,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188923 2024-11-13T14:29:48,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188923 2024-11-13T14:29:48,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188923) roll requested 2024-11-13T14:29:48,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188930 2024-11-13T14:29:48,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188930 2024-11-13T14:29:48,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188930) roll requested 2024-11-13T14:29:48,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188938 2024-11-13T14:29:48,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188938 2024-11-13T14:29:48,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188938) roll requested 2024-11-13T14:29:48,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188947 2024-11-13T14:29:48,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188947 2024-11-13T14:29:48,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188947) roll requested 2024-11-13T14:29:48,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188955 2024-11-13T14:29:48,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188955 2024-11-13T14:29:48,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188955) roll requested 2024-11-13T14:29:48,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188965 2024-11-13T14:29:48,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188965 2024-11-13T14:29:48,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188965) roll requested 2024-11-13T14:29:48,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188975 2024-11-13T14:29:48,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188975 2024-11-13T14:29:48,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188975) roll requested 2024-11-13T14:29:48,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188985 2024-11-13T14:29:48,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:48,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188985 2024-11-13T14:29:48,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:48,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:48,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:48,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:48,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188985) roll requested 2024-11-13T14:29:48,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508188995 2024-11-13T14:29:49,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188995 2024-11-13T14:29:49,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508188995) roll requested 2024-11-13T14:29:49,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189003 2024-11-13T14:29:49,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508188995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189003 2024-11-13T14:29:49,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189003) roll requested 2024-11-13T14:29:49,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189015 2024-11-13T14:29:49,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189015 2024-11-13T14:29:49,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189015) roll requested 2024-11-13T14:29:49,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189023 2024-11-13T14:29:49,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189023 2024-11-13T14:29:49,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189023) roll requested 2024-11-13T14:29:49,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189031 2024-11-13T14:29:49,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189031 2024-11-13T14:29:49,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189031) roll requested 2024-11-13T14:29:49,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189040 2024-11-13T14:29:49,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189040 2024-11-13T14:29:49,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189040) roll requested 2024-11-13T14:29:49,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189050 2024-11-13T14:29:49,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189050 2024-11-13T14:29:49,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189050) roll requested 2024-11-13T14:29:49,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189060 2024-11-13T14:29:49,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189060 2024-11-13T14:29:49,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189060) roll requested 2024-11-13T14:29:49,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189069 2024-11-13T14:29:49,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189069 2024-11-13T14:29:49,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189069) roll requested 2024-11-13T14:29:49,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189079 2024-11-13T14:29:49,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189079 2024-11-13T14:29:49,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189079) roll requested 2024-11-13T14:29:49,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189086 2024-11-13T14:29:49,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189086 2024-11-13T14:29:49,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189086) roll requested 2024-11-13T14:29:49,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189093 2024-11-13T14:29:49,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189093 2024-11-13T14:29:49,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189093) roll requested 2024-11-13T14:29:49,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189101 2024-11-13T14:29:49,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189101 2024-11-13T14:29:49,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189101) roll requested 2024-11-13T14:29:49,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189110 2024-11-13T14:29:49,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189110 2024-11-13T14:29:49,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189110) roll requested 2024-11-13T14:29:49,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189119 2024-11-13T14:29:49,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189119 2024-11-13T14:29:49,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189119) roll requested 2024-11-13T14:29:49,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189128 2024-11-13T14:29:49,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189128 2024-11-13T14:29:49,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189128) roll requested 2024-11-13T14:29:49,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189138 2024-11-13T14:29:49,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189138 2024-11-13T14:29:49,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189138) roll requested 2024-11-13T14:29:49,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189146 2024-11-13T14:29:49,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189146 2024-11-13T14:29:49,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189146) roll requested 2024-11-13T14:29:49,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189151 2024-11-13T14:29:49,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189151 2024-11-13T14:29:49,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189151) roll requested 2024-11-13T14:29:49,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189156 2024-11-13T14:29:49,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189156 2024-11-13T14:29:49,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189156) roll requested 2024-11-13T14:29:49,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189162 2024-11-13T14:29:49,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189162 2024-11-13T14:29:49,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189162) roll requested 2024-11-13T14:29:49,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189169 2024-11-13T14:29:49,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189169 2024-11-13T14:29:49,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189169) roll requested 2024-11-13T14:29:49,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189176 2024-11-13T14:29:49,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189176 2024-11-13T14:29:49,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189176) roll requested 2024-11-13T14:29:49,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189184 2024-11-13T14:29:49,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189184 2024-11-13T14:29:49,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189184) roll requested 2024-11-13T14:29:49,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189193 2024-11-13T14:29:49,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189193 2024-11-13T14:29:49,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189193) roll requested 2024-11-13T14:29:49,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189201 2024-11-13T14:29:49,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189201 2024-11-13T14:29:49,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189201) roll requested 2024-11-13T14:29:49,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189209 2024-11-13T14:29:49,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189209 2024-11-13T14:29:49,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189209) roll requested 2024-11-13T14:29:49,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189217 2024-11-13T14:29:49,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189217 2024-11-13T14:29:49,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189217) roll requested 2024-11-13T14:29:49,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189225 2024-11-13T14:29:49,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189225 2024-11-13T14:29:49,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189225) roll requested 2024-11-13T14:29:49,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189234 2024-11-13T14:29:49,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189234 2024-11-13T14:29:49,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189234) roll requested 2024-11-13T14:29:49,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189243 2024-11-13T14:29:49,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189243 2024-11-13T14:29:49,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189243) roll requested 2024-11-13T14:29:49,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189253 2024-11-13T14:29:49,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189253 2024-11-13T14:29:49,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189253) roll requested 2024-11-13T14:29:49,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189262 2024-11-13T14:29:49,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189262 2024-11-13T14:29:49,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189262) roll requested 2024-11-13T14:29:49,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189272 2024-11-13T14:29:49,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189272 2024-11-13T14:29:49,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189272) roll requested 2024-11-13T14:29:49,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189283 2024-11-13T14:29:49,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189283 2024-11-13T14:29:49,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189283) roll requested 2024-11-13T14:29:49,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189293 2024-11-13T14:29:49,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189293 2024-11-13T14:29:49,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189293) roll requested 2024-11-13T14:29:49,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189300 2024-11-13T14:29:49,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189300 2024-11-13T14:29:49,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189300) roll requested 2024-11-13T14:29:49,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189308 2024-11-13T14:29:49,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189308 2024-11-13T14:29:49,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189308) roll requested 2024-11-13T14:29:49,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189318 2024-11-13T14:29:49,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189318 2024-11-13T14:29:49,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189318) roll requested 2024-11-13T14:29:49,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189327 2024-11-13T14:29:49,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189327 2024-11-13T14:29:49,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189327) roll requested 2024-11-13T14:29:49,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189333 2024-11-13T14:29:49,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189333 2024-11-13T14:29:49,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189333) roll requested 2024-11-13T14:29:49,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189340 2024-11-13T14:29:49,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189340 2024-11-13T14:29:49,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189340) roll requested 2024-11-13T14:29:49,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189347 2024-11-13T14:29:49,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189347 2024-11-13T14:29:49,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189347) roll requested 2024-11-13T14:29:49,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189355 2024-11-13T14:29:49,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189355 2024-11-13T14:29:49,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189355) roll requested 2024-11-13T14:29:49,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189362 2024-11-13T14:29:49,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189362 2024-11-13T14:29:49,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189362) roll requested 2024-11-13T14:29:49,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189369 2024-11-13T14:29:49,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189369 2024-11-13T14:29:49,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189369) roll requested 2024-11-13T14:29:49,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189378 2024-11-13T14:29:49,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189378 2024-11-13T14:29:49,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189378) roll requested 2024-11-13T14:29:49,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189387 2024-11-13T14:29:49,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189387 2024-11-13T14:29:49,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189387) roll requested 2024-11-13T14:29:49,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189396 2024-11-13T14:29:49,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189396 2024-11-13T14:29:49,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189396) roll requested 2024-11-13T14:29:49,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189402 2024-11-13T14:29:49,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189402 2024-11-13T14:29:49,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189402) roll requested 2024-11-13T14:29:49,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189410 2024-11-13T14:29:49,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189410 2024-11-13T14:29:49,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189410) roll requested 2024-11-13T14:29:49,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189418 2024-11-13T14:29:49,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189418 2024-11-13T14:29:49,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189418) roll requested 2024-11-13T14:29:49,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189426 2024-11-13T14:29:49,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189426 2024-11-13T14:29:49,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189426) roll requested 2024-11-13T14:29:49,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189434 2024-11-13T14:29:49,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189434 2024-11-13T14:29:49,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189434) roll requested 2024-11-13T14:29:49,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189444 2024-11-13T14:29:49,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189444 2024-11-13T14:29:49,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189444) roll requested 2024-11-13T14:29:49,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189454 2024-11-13T14:29:49,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189454 2024-11-13T14:29:49,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189454) roll requested 2024-11-13T14:29:49,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189465 2024-11-13T14:29:49,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189465 2024-11-13T14:29:49,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189465) roll requested 2024-11-13T14:29:49,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189475 2024-11-13T14:29:49,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189475 2024-11-13T14:29:49,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189475) roll requested 2024-11-13T14:29:49,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189484 2024-11-13T14:29:49,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189484 2024-11-13T14:29:49,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189484) roll requested 2024-11-13T14:29:49,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189493 2024-11-13T14:29:49,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189493 2024-11-13T14:29:49,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189493) roll requested 2024-11-13T14:29:49,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189502 2024-11-13T14:29:49,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189502 2024-11-13T14:29:49,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189502) roll requested 2024-11-13T14:29:49,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189508 2024-11-13T14:29:49,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189508 2024-11-13T14:29:49,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189508) roll requested 2024-11-13T14:29:49,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189515 2024-11-13T14:29:49,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189515 2024-11-13T14:29:49,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189515) roll requested 2024-11-13T14:29:49,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189523 2024-11-13T14:29:49,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189523 2024-11-13T14:29:49,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189523) roll requested 2024-11-13T14:29:49,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189532 2024-11-13T14:29:49,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189532 2024-11-13T14:29:49,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189532) roll requested 2024-11-13T14:29:49,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189542 2024-11-13T14:29:49,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189542 2024-11-13T14:29:49,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189542) roll requested 2024-11-13T14:29:49,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189550 2024-11-13T14:29:49,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189550 2024-11-13T14:29:49,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189550) roll requested 2024-11-13T14:29:49,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189557 2024-11-13T14:29:49,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189557 2024-11-13T14:29:49,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189557) roll requested 2024-11-13T14:29:49,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189565 2024-11-13T14:29:49,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189565 2024-11-13T14:29:49,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189565) roll requested 2024-11-13T14:29:49,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189573 2024-11-13T14:29:49,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189573 2024-11-13T14:29:49,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189573) roll requested 2024-11-13T14:29:49,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189583 2024-11-13T14:29:49,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189583 2024-11-13T14:29:49,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189583) roll requested 2024-11-13T14:29:49,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189593 2024-11-13T14:29:49,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189593 2024-11-13T14:29:49,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189593) roll requested 2024-11-13T14:29:49,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189602 2024-11-13T14:29:49,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189602 2024-11-13T14:29:49,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189602) roll requested 2024-11-13T14:29:49,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189609 2024-11-13T14:29:49,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189609 2024-11-13T14:29:49,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189609) roll requested 2024-11-13T14:29:49,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189617 2024-11-13T14:29:49,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189617 2024-11-13T14:29:49,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189617) roll requested 2024-11-13T14:29:49,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189626 2024-11-13T14:29:49,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189626 2024-11-13T14:29:49,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189626) roll requested 2024-11-13T14:29:49,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189636 2024-11-13T14:29:49,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189636 2024-11-13T14:29:49,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189636) roll requested 2024-11-13T14:29:49,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189645 2024-11-13T14:29:49,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189645 2024-11-13T14:29:49,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189645) roll requested 2024-11-13T14:29:49,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189653 2024-11-13T14:29:49,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189653 2024-11-13T14:29:49,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189653) roll requested 2024-11-13T14:29:49,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189660 2024-11-13T14:29:49,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189660 2024-11-13T14:29:49,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189660) roll requested 2024-11-13T14:29:49,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189667 2024-11-13T14:29:49,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189667 2024-11-13T14:29:49,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189667) roll requested 2024-11-13T14:29:49,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189676 2024-11-13T14:29:49,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189676 2024-11-13T14:29:49,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189676) roll requested 2024-11-13T14:29:49,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189684 2024-11-13T14:29:49,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189684 2024-11-13T14:29:49,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189684) roll requested 2024-11-13T14:29:49,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189692 2024-11-13T14:29:49,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189692 2024-11-13T14:29:49,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189692) roll requested 2024-11-13T14:29:49,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189701 2024-11-13T14:29:49,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189701 2024-11-13T14:29:49,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189701) roll requested 2024-11-13T14:29:49,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189710 2024-11-13T14:29:49,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189710 2024-11-13T14:29:49,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189710) roll requested 2024-11-13T14:29:49,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189719 2024-11-13T14:29:49,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189719 2024-11-13T14:29:49,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189719) roll requested 2024-11-13T14:29:49,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189730 2024-11-13T14:29:49,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189730 2024-11-13T14:29:49,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189730) roll requested 2024-11-13T14:29:49,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189740 2024-11-13T14:29:49,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189740 2024-11-13T14:29:49,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189740) roll requested 2024-11-13T14:29:49,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189751 2024-11-13T14:29:49,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189751 2024-11-13T14:29:49,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189751) roll requested 2024-11-13T14:29:49,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189759 2024-11-13T14:29:49,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189759 2024-11-13T14:29:49,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189759) roll requested 2024-11-13T14:29:49,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189768 2024-11-13T14:29:49,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189768 2024-11-13T14:29:49,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189768) roll requested 2024-11-13T14:29:49,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189778 2024-11-13T14:29:49,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189778 2024-11-13T14:29:49,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189778) roll requested 2024-11-13T14:29:49,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189786 2024-11-13T14:29:49,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189786 2024-11-13T14:29:49,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189786) roll requested 2024-11-13T14:29:49,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189794 2024-11-13T14:29:49,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189794 2024-11-13T14:29:49,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189794) roll requested 2024-11-13T14:29:49,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189802 2024-11-13T14:29:49,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189802 2024-11-13T14:29:49,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189802) roll requested 2024-11-13T14:29:49,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189811 2024-11-13T14:29:49,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189811 2024-11-13T14:29:49,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189811) roll requested 2024-11-13T14:29:49,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189822 2024-11-13T14:29:49,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189822 2024-11-13T14:29:49,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189822) roll requested 2024-11-13T14:29:49,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189831 2024-11-13T14:29:49,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189831 2024-11-13T14:29:49,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189831) roll requested 2024-11-13T14:29:49,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189841 2024-11-13T14:29:49,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189841 2024-11-13T14:29:49,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189841) roll requested 2024-11-13T14:29:49,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189851 2024-11-13T14:29:49,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189851 2024-11-13T14:29:49,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189851) roll requested 2024-11-13T14:29:49,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189859 2024-11-13T14:29:49,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189859 2024-11-13T14:29:49,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189859) roll requested 2024-11-13T14:29:49,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189868 2024-11-13T14:29:49,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189868 2024-11-13T14:29:49,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189868) roll requested 2024-11-13T14:29:49,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189876 2024-11-13T14:29:49,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189876 2024-11-13T14:29:49,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189876) roll requested 2024-11-13T14:29:49,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189885 2024-11-13T14:29:49,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189885 2024-11-13T14:29:49,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189885) roll requested 2024-11-13T14:29:49,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189892 2024-11-13T14:29:49,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189892 2024-11-13T14:29:49,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189892) roll requested 2024-11-13T14:29:49,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189897 2024-11-13T14:29:49,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189897 2024-11-13T14:29:49,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189897) roll requested 2024-11-13T14:29:49,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189904 2024-11-13T14:29:49,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189904 2024-11-13T14:29:49,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189904) roll requested 2024-11-13T14:29:49,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189912 2024-11-13T14:29:49,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189912 2024-11-13T14:29:49,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189912) roll requested 2024-11-13T14:29:49,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189920 2024-11-13T14:29:49,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189920 2024-11-13T14:29:49,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189920) roll requested 2024-11-13T14:29:49,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189929 2024-11-13T14:29:49,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189929 2024-11-13T14:29:49,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189929) roll requested 2024-11-13T14:29:49,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189937 2024-11-13T14:29:49,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189937 2024-11-13T14:29:49,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189937) roll requested 2024-11-13T14:29:49,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189946 2024-11-13T14:29:49,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189946 2024-11-13T14:29:49,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189946) roll requested 2024-11-13T14:29:49,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189957 2024-11-13T14:29:49,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189957 2024-11-13T14:29:49,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189957) roll requested 2024-11-13T14:29:49,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189966 2024-11-13T14:29:49,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189966 2024-11-13T14:29:49,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189966) roll requested 2024-11-13T14:29:49,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189972 2024-11-13T14:29:49,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189972 2024-11-13T14:29:49,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189972) roll requested 2024-11-13T14:29:49,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189979 2024-11-13T14:29:49,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189979 2024-11-13T14:29:49,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189979) roll requested 2024-11-13T14:29:49,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189986 2024-11-13T14:29:49,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189986 2024-11-13T14:29:49,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189986) roll requested 2024-11-13T14:29:49,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189992 2024-11-13T14:29:49,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:49,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189992 2024-11-13T14:29:49,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:49,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:49,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:49,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:49,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189992) roll requested 2024-11-13T14:29:49,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508189997 2024-11-13T14:29:50,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189997 2024-11-13T14:29:50,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508189997) roll requested 2024-11-13T14:29:50,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190003 2024-11-13T14:29:50,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508189997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190003 2024-11-13T14:29:50,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190003) roll requested 2024-11-13T14:29:50,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190009 2024-11-13T14:29:50,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190009 2024-11-13T14:29:50,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190009) roll requested 2024-11-13T14:29:50,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190015 2024-11-13T14:29:50,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190015 2024-11-13T14:29:50,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190015) roll requested 2024-11-13T14:29:50,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190020 2024-11-13T14:29:50,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190020 2024-11-13T14:29:50,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190020) roll requested 2024-11-13T14:29:50,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190026 2024-11-13T14:29:50,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190026 2024-11-13T14:29:50,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190026) roll requested 2024-11-13T14:29:50,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190033 2024-11-13T14:29:50,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190033 2024-11-13T14:29:50,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190033) roll requested 2024-11-13T14:29:50,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190040 2024-11-13T14:29:50,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190040 2024-11-13T14:29:50,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190040) roll requested 2024-11-13T14:29:50,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190049 2024-11-13T14:29:50,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190049 2024-11-13T14:29:50,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190049) roll requested 2024-11-13T14:29:50,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190058 2024-11-13T14:29:50,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190058 2024-11-13T14:29:50,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190058) roll requested 2024-11-13T14:29:50,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190067 2024-11-13T14:29:50,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190067 2024-11-13T14:29:50,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190067) roll requested 2024-11-13T14:29:50,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190074 2024-11-13T14:29:50,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190074 2024-11-13T14:29:50,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190074) roll requested 2024-11-13T14:29:50,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190082 2024-11-13T14:29:50,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190082 2024-11-13T14:29:50,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190082) roll requested 2024-11-13T14:29:50,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190090 2024-11-13T14:29:50,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190090 2024-11-13T14:29:50,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190090) roll requested 2024-11-13T14:29:50,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190099 2024-11-13T14:29:50,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190099 2024-11-13T14:29:50,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190099) roll requested 2024-11-13T14:29:50,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190105 2024-11-13T14:29:50,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190105 2024-11-13T14:29:50,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190105) roll requested 2024-11-13T14:29:50,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190113 2024-11-13T14:29:50,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190113 2024-11-13T14:29:50,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190113) roll requested 2024-11-13T14:29:50,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190121 2024-11-13T14:29:50,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190121 2024-11-13T14:29:50,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190121) roll requested 2024-11-13T14:29:50,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190130 2024-11-13T14:29:50,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190130 2024-11-13T14:29:50,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190130) roll requested 2024-11-13T14:29:50,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190141 2024-11-13T14:29:50,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190141 2024-11-13T14:29:50,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190141) roll requested 2024-11-13T14:29:50,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190150 2024-11-13T14:29:50,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190150 2024-11-13T14:29:50,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190150) roll requested 2024-11-13T14:29:50,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190160 2024-11-13T14:29:50,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190160 2024-11-13T14:29:50,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190160) roll requested 2024-11-13T14:29:50,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190170 2024-11-13T14:29:50,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190170 2024-11-13T14:29:50,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190170) roll requested 2024-11-13T14:29:50,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190178 2024-11-13T14:29:50,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190178 2024-11-13T14:29:50,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190178) roll requested 2024-11-13T14:29:50,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190187 2024-11-13T14:29:50,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190187 2024-11-13T14:29:50,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190187) roll requested 2024-11-13T14:29:50,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190196 2024-11-13T14:29:50,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190196 2024-11-13T14:29:50,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190196) roll requested 2024-11-13T14:29:50,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190203 2024-11-13T14:29:50,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190203 2024-11-13T14:29:50,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190203) roll requested 2024-11-13T14:29:50,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190211 2024-11-13T14:29:50,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190211 2024-11-13T14:29:50,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190211) roll requested 2024-11-13T14:29:50,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190220 2024-11-13T14:29:50,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190220 2024-11-13T14:29:50,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190220) roll requested 2024-11-13T14:29:50,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190229 2024-11-13T14:29:50,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190229 2024-11-13T14:29:50,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190229) roll requested 2024-11-13T14:29:50,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190237 2024-11-13T14:29:50,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190237 2024-11-13T14:29:50,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190237) roll requested 2024-11-13T14:29:50,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190246 2024-11-13T14:29:50,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190246 2024-11-13T14:29:50,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190246) roll requested 2024-11-13T14:29:50,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190255 2024-11-13T14:29:50,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190255 2024-11-13T14:29:50,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190255) roll requested 2024-11-13T14:29:50,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190264 2024-11-13T14:29:50,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190264 2024-11-13T14:29:50,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190264) roll requested 2024-11-13T14:29:50,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190272 2024-11-13T14:29:50,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190272 2024-11-13T14:29:50,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190272) roll requested 2024-11-13T14:29:50,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190278 2024-11-13T14:29:50,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190278 2024-11-13T14:29:50,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190278) roll requested 2024-11-13T14:29:50,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190284 2024-11-13T14:29:50,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190284 2024-11-13T14:29:50,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190284) roll requested 2024-11-13T14:29:50,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190293 2024-11-13T14:29:50,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190293 2024-11-13T14:29:50,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190293) roll requested 2024-11-13T14:29:50,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190302 2024-11-13T14:29:50,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190302 2024-11-13T14:29:50,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190302) roll requested 2024-11-13T14:29:50,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190312 2024-11-13T14:29:50,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190312 2024-11-13T14:29:50,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190312) roll requested 2024-11-13T14:29:50,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190320 2024-11-13T14:29:50,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190320 2024-11-13T14:29:50,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190320) roll requested 2024-11-13T14:29:50,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190327 2024-11-13T14:29:50,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190327 2024-11-13T14:29:50,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190327) roll requested 2024-11-13T14:29:50,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190336 2024-11-13T14:29:50,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190336 2024-11-13T14:29:50,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190336) roll requested 2024-11-13T14:29:50,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190344 2024-11-13T14:29:50,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190344 2024-11-13T14:29:50,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190344) roll requested 2024-11-13T14:29:50,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190353 2024-11-13T14:29:50,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190353 2024-11-13T14:29:50,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190353) roll requested 2024-11-13T14:29:50,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190363 2024-11-13T14:29:50,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190363 2024-11-13T14:29:50,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190363) roll requested 2024-11-13T14:29:50,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190373 2024-11-13T14:29:50,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190373 2024-11-13T14:29:50,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190373) roll requested 2024-11-13T14:29:50,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190381 2024-11-13T14:29:50,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190381 2024-11-13T14:29:50,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190381) roll requested 2024-11-13T14:29:50,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190391 2024-11-13T14:29:50,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190391 2024-11-13T14:29:50,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190391) roll requested 2024-11-13T14:29:50,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190401 2024-11-13T14:29:50,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190401 2024-11-13T14:29:50,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190401) roll requested 2024-11-13T14:29:50,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190411 2024-11-13T14:29:50,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190411 2024-11-13T14:29:50,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190411) roll requested 2024-11-13T14:29:50,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190422 2024-11-13T14:29:50,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190422 2024-11-13T14:29:50,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190422) roll requested 2024-11-13T14:29:50,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190430 2024-11-13T14:29:50,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190430 2024-11-13T14:29:50,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190430) roll requested 2024-11-13T14:29:50,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190438 2024-11-13T14:29:50,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190438 2024-11-13T14:29:50,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190438) roll requested 2024-11-13T14:29:50,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190448 2024-11-13T14:29:50,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190448 2024-11-13T14:29:50,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190448) roll requested 2024-11-13T14:29:50,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190456 2024-11-13T14:29:50,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190456 2024-11-13T14:29:50,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190456) roll requested 2024-11-13T14:29:50,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190462 2024-11-13T14:29:50,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190462 2024-11-13T14:29:50,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190462) roll requested 2024-11-13T14:29:50,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190470 2024-11-13T14:29:50,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190470 2024-11-13T14:29:50,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190470) roll requested 2024-11-13T14:29:50,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190477 2024-11-13T14:29:50,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190477 2024-11-13T14:29:50,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190477) roll requested 2024-11-13T14:29:50,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190483 2024-11-13T14:29:50,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190483 2024-11-13T14:29:50,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190483) roll requested 2024-11-13T14:29:50,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190488 2024-11-13T14:29:50,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190488 2024-11-13T14:29:50,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190488) roll requested 2024-11-13T14:29:50,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190495 2024-11-13T14:29:50,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190495 2024-11-13T14:29:50,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190495) roll requested 2024-11-13T14:29:50,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190504 2024-11-13T14:29:50,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190504 2024-11-13T14:29:50,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190504) roll requested 2024-11-13T14:29:50,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190512 2024-11-13T14:29:50,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190512 2024-11-13T14:29:50,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190512) roll requested 2024-11-13T14:29:50,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190518 2024-11-13T14:29:50,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190518 2024-11-13T14:29:50,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190518) roll requested 2024-11-13T14:29:50,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190524 2024-11-13T14:29:50,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190524 2024-11-13T14:29:50,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190524) roll requested 2024-11-13T14:29:50,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190531 2024-11-13T14:29:50,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190531 2024-11-13T14:29:50,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190531) roll requested 2024-11-13T14:29:50,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190539 2024-11-13T14:29:50,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190539 2024-11-13T14:29:50,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190539) roll requested 2024-11-13T14:29:50,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190547 2024-11-13T14:29:50,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190547 2024-11-13T14:29:50,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190547) roll requested 2024-11-13T14:29:50,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190553 2024-11-13T14:29:50,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190553 2024-11-13T14:29:50,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190553) roll requested 2024-11-13T14:29:50,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190558 2024-11-13T14:29:50,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190558 2024-11-13T14:29:50,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190558) roll requested 2024-11-13T14:29:50,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190564 2024-11-13T14:29:50,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190564 2024-11-13T14:29:50,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190564) roll requested 2024-11-13T14:29:50,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190571 2024-11-13T14:29:50,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190571 2024-11-13T14:29:50,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190571) roll requested 2024-11-13T14:29:50,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190580 2024-11-13T14:29:50,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190580 2024-11-13T14:29:50,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190580) roll requested 2024-11-13T14:29:50,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190589 2024-11-13T14:29:50,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190589 2024-11-13T14:29:50,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190589) roll requested 2024-11-13T14:29:50,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190600 2024-11-13T14:29:50,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190600 2024-11-13T14:29:50,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190600) roll requested 2024-11-13T14:29:50,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190609 2024-11-13T14:29:50,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190609 2024-11-13T14:29:50,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190609) roll requested 2024-11-13T14:29:50,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190617 2024-11-13T14:29:50,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190617 2024-11-13T14:29:50,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190617) roll requested 2024-11-13T14:29:50,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190624 2024-11-13T14:29:50,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190624 2024-11-13T14:29:50,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190624) roll requested 2024-11-13T14:29:50,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190632 2024-11-13T14:29:50,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190632 2024-11-13T14:29:50,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190632) roll requested 2024-11-13T14:29:50,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190641 2024-11-13T14:29:50,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190641 2024-11-13T14:29:50,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190641) roll requested 2024-11-13T14:29:50,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190652 2024-11-13T14:29:50,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190652 2024-11-13T14:29:50,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190652) roll requested 2024-11-13T14:29:50,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190663 2024-11-13T14:29:50,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190663 2024-11-13T14:29:50,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190663) roll requested 2024-11-13T14:29:50,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190672 2024-11-13T14:29:50,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190672 2024-11-13T14:29:50,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190672) roll requested 2024-11-13T14:29:50,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190729 2024-11-13T14:29:50,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190729 2024-11-13T14:29:50,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190729) roll requested 2024-11-13T14:29:50,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190740 2024-11-13T14:29:50,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190740 2024-11-13T14:29:50,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190740) roll requested 2024-11-13T14:29:50,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190750 2024-11-13T14:29:50,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190750 2024-11-13T14:29:50,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190750) roll requested 2024-11-13T14:29:50,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190761 2024-11-13T14:29:50,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190761 2024-11-13T14:29:50,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190761) roll requested 2024-11-13T14:29:50,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190772 2024-11-13T14:29:50,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190772 2024-11-13T14:29:50,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190772) roll requested 2024-11-13T14:29:50,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190781 2024-11-13T14:29:50,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190781 2024-11-13T14:29:50,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190781) roll requested 2024-11-13T14:29:50,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190789 2024-11-13T14:29:50,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190789 2024-11-13T14:29:50,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190789) roll requested 2024-11-13T14:29:50,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190798 2024-11-13T14:29:50,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190798 2024-11-13T14:29:50,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190798) roll requested 2024-11-13T14:29:50,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190808 2024-11-13T14:29:50,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190808 2024-11-13T14:29:50,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190808) roll requested 2024-11-13T14:29:50,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190818 2024-11-13T14:29:50,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190818 2024-11-13T14:29:50,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190818) roll requested 2024-11-13T14:29:50,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190827 2024-11-13T14:29:50,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190827 2024-11-13T14:29:50,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190827) roll requested 2024-11-13T14:29:50,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190837 2024-11-13T14:29:50,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190837 2024-11-13T14:29:50,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190837) roll requested 2024-11-13T14:29:50,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190847 2024-11-13T14:29:50,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190847 2024-11-13T14:29:50,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190847) roll requested 2024-11-13T14:29:50,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190855 2024-11-13T14:29:50,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190855 2024-11-13T14:29:50,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190855) roll requested 2024-11-13T14:29:50,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190864 2024-11-13T14:29:50,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190864 2024-11-13T14:29:50,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190864) roll requested 2024-11-13T14:29:50,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190874 2024-11-13T14:29:50,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190874 2024-11-13T14:29:50,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190874) roll requested 2024-11-13T14:29:50,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190882 2024-11-13T14:29:50,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190882 2024-11-13T14:29:50,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190882) roll requested 2024-11-13T14:29:50,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190890 2024-11-13T14:29:50,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190890 2024-11-13T14:29:50,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190890) roll requested 2024-11-13T14:29:50,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190899 2024-11-13T14:29:50,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190899 2024-11-13T14:29:50,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190899) roll requested 2024-11-13T14:29:50,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190907 2024-11-13T14:29:50,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190907 2024-11-13T14:29:50,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190907) roll requested 2024-11-13T14:29:50,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190915 2024-11-13T14:29:50,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190915 2024-11-13T14:29:50,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190915) roll requested 2024-11-13T14:29:50,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190924 2024-11-13T14:29:50,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190924 2024-11-13T14:29:50,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190924) roll requested 2024-11-13T14:29:50,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190933 2024-11-13T14:29:50,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190933 2024-11-13T14:29:50,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190933) roll requested 2024-11-13T14:29:50,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190939 2024-11-13T14:29:50,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190939 2024-11-13T14:29:50,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190939) roll requested 2024-11-13T14:29:50,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190945 2024-11-13T14:29:50,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190945 2024-11-13T14:29:50,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190945) roll requested 2024-11-13T14:29:50,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190954 2024-11-13T14:29:50,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190954 2024-11-13T14:29:50,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190954) roll requested 2024-11-13T14:29:50,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190962 2024-11-13T14:29:50,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190962 2024-11-13T14:29:50,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190962) roll requested 2024-11-13T14:29:50,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190971 2024-11-13T14:29:50,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190971 2024-11-13T14:29:50,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190971) roll requested 2024-11-13T14:29:50,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190979 2024-11-13T14:29:50,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190979 2024-11-13T14:29:50,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190979) roll requested 2024-11-13T14:29:50,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190987 2024-11-13T14:29:50,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:50,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190987 2024-11-13T14:29:50,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:50,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:50,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:50,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:50,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190987) roll requested 2024-11-13T14:29:50,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508190995 2024-11-13T14:29:51,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190995 2024-11-13T14:29:51,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508190995) roll requested 2024-11-13T14:29:51,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191003 2024-11-13T14:29:51,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508190995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191003 2024-11-13T14:29:51,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191003) roll requested 2024-11-13T14:29:51,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191010 2024-11-13T14:29:51,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191010 2024-11-13T14:29:51,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191010) roll requested 2024-11-13T14:29:51,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191017 2024-11-13T14:29:51,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191017 2024-11-13T14:29:51,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191017) roll requested 2024-11-13T14:29:51,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191024 2024-11-13T14:29:51,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191024 2024-11-13T14:29:51,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191024) roll requested 2024-11-13T14:29:51,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191032 2024-11-13T14:29:51,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191032 2024-11-13T14:29:51,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191032) roll requested 2024-11-13T14:29:51,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191041 2024-11-13T14:29:51,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191041 2024-11-13T14:29:51,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191041) roll requested 2024-11-13T14:29:51,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191050 2024-11-13T14:29:51,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191050 2024-11-13T14:29:51,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191050) roll requested 2024-11-13T14:29:51,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191060 2024-11-13T14:29:51,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191060 2024-11-13T14:29:51,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191060) roll requested 2024-11-13T14:29:51,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191069 2024-11-13T14:29:51,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191069 2024-11-13T14:29:51,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191069) roll requested 2024-11-13T14:29:51,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191080 2024-11-13T14:29:51,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191080 2024-11-13T14:29:51,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191080) roll requested 2024-11-13T14:29:51,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191090 2024-11-13T14:29:51,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191090 2024-11-13T14:29:51,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191090) roll requested 2024-11-13T14:29:51,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191097 2024-11-13T14:29:51,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191097 2024-11-13T14:29:51,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191097) roll requested 2024-11-13T14:29:51,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191105 2024-11-13T14:29:51,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191105 2024-11-13T14:29:51,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191105) roll requested 2024-11-13T14:29:51,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191112 2024-11-13T14:29:51,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191112 2024-11-13T14:29:51,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191112) roll requested 2024-11-13T14:29:51,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191118 2024-11-13T14:29:51,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191118 2024-11-13T14:29:51,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191118) roll requested 2024-11-13T14:29:51,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191126 2024-11-13T14:29:51,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191126 2024-11-13T14:29:51,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191126) roll requested 2024-11-13T14:29:51,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191134 2024-11-13T14:29:51,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191134 2024-11-13T14:29:51,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191134) roll requested 2024-11-13T14:29:51,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191144 2024-11-13T14:29:51,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191144 2024-11-13T14:29:51,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191144) roll requested 2024-11-13T14:29:51,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191153 2024-11-13T14:29:51,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191153 2024-11-13T14:29:51,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191153) roll requested 2024-11-13T14:29:51,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191159 2024-11-13T14:29:51,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191159 2024-11-13T14:29:51,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191159) roll requested 2024-11-13T14:29:51,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191166 2024-11-13T14:29:51,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191166 2024-11-13T14:29:51,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191166) roll requested 2024-11-13T14:29:51,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191174 2024-11-13T14:29:51,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191174 2024-11-13T14:29:51,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191174) roll requested 2024-11-13T14:29:51,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191181 2024-11-13T14:29:51,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191181 2024-11-13T14:29:51,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191181) roll requested 2024-11-13T14:29:51,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191188 2024-11-13T14:29:51,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191188 2024-11-13T14:29:51,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191188) roll requested 2024-11-13T14:29:51,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191196 2024-11-13T14:29:51,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191196 2024-11-13T14:29:51,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191196) roll requested 2024-11-13T14:29:51,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191205 2024-11-13T14:29:51,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191205 2024-11-13T14:29:51,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191205) roll requested 2024-11-13T14:29:51,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191215 2024-11-13T14:29:51,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191215 2024-11-13T14:29:51,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191215) roll requested 2024-11-13T14:29:51,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191226 2024-11-13T14:29:51,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191226 2024-11-13T14:29:51,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191226) roll requested 2024-11-13T14:29:51,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191235 2024-11-13T14:29:51,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191235 2024-11-13T14:29:51,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191235) roll requested 2024-11-13T14:29:51,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191245 2024-11-13T14:29:51,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191245 2024-11-13T14:29:51,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191245) roll requested 2024-11-13T14:29:51,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191255 2024-11-13T14:29:51,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191255 2024-11-13T14:29:51,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191255) roll requested 2024-11-13T14:29:51,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191265 2024-11-13T14:29:51,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191265 2024-11-13T14:29:51,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191265) roll requested 2024-11-13T14:29:51,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191272 2024-11-13T14:29:51,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191272 2024-11-13T14:29:51,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191272) roll requested 2024-11-13T14:29:51,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191280 2024-11-13T14:29:51,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191280 2024-11-13T14:29:51,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191280) roll requested 2024-11-13T14:29:51,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191288 2024-11-13T14:29:51,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191288 2024-11-13T14:29:51,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191288) roll requested 2024-11-13T14:29:51,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191295 2024-11-13T14:29:51,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191295 2024-11-13T14:29:51,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191295) roll requested 2024-11-13T14:29:51,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191303 2024-11-13T14:29:51,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191303 2024-11-13T14:29:51,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191303) roll requested 2024-11-13T14:29:51,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191311 2024-11-13T14:29:51,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191311 2024-11-13T14:29:51,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191311) roll requested 2024-11-13T14:29:51,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191319 2024-11-13T14:29:51,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191319 2024-11-13T14:29:51,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191319) roll requested 2024-11-13T14:29:51,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191327 2024-11-13T14:29:51,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191327 2024-11-13T14:29:51,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191327) roll requested 2024-11-13T14:29:51,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191337 2024-11-13T14:29:51,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191337 2024-11-13T14:29:51,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191337) roll requested 2024-11-13T14:29:51,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191346 2024-11-13T14:29:51,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191346 2024-11-13T14:29:51,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191346) roll requested 2024-11-13T14:29:51,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191354 2024-11-13T14:29:51,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191354 2024-11-13T14:29:51,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191354) roll requested 2024-11-13T14:29:51,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191365 2024-11-13T14:29:51,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191365 2024-11-13T14:29:51,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191365) roll requested 2024-11-13T14:29:51,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191375 2024-11-13T14:29:51,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191375 2024-11-13T14:29:51,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191375) roll requested 2024-11-13T14:29:51,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191385 2024-11-13T14:29:51,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191385 2024-11-13T14:29:51,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191385) roll requested 2024-11-13T14:29:51,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191391 2024-11-13T14:29:51,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191391 2024-11-13T14:29:51,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191391) roll requested 2024-11-13T14:29:51,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191397 2024-11-13T14:29:51,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191397 2024-11-13T14:29:51,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191397) roll requested 2024-11-13T14:29:51,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191405 2024-11-13T14:29:51,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191405 2024-11-13T14:29:51,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191405) roll requested 2024-11-13T14:29:51,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191414 2024-11-13T14:29:51,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191414 2024-11-13T14:29:51,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191414) roll requested 2024-11-13T14:29:51,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191424 2024-11-13T14:29:51,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191424 2024-11-13T14:29:51,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191424) roll requested 2024-11-13T14:29:51,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191434 2024-11-13T14:29:51,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191434 2024-11-13T14:29:51,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191434) roll requested 2024-11-13T14:29:51,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191442 2024-11-13T14:29:51,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191442 2024-11-13T14:29:51,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191442) roll requested 2024-11-13T14:29:51,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191452 2024-11-13T14:29:51,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191452 2024-11-13T14:29:51,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191452) roll requested 2024-11-13T14:29:51,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191461 2024-11-13T14:29:51,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191461 2024-11-13T14:29:51,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191461) roll requested 2024-11-13T14:29:51,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191468 2024-11-13T14:29:51,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191468 2024-11-13T14:29:51,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191468) roll requested 2024-11-13T14:29:51,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191476 2024-11-13T14:29:51,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191476 2024-11-13T14:29:51,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191476) roll requested 2024-11-13T14:29:51,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191486 2024-11-13T14:29:51,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191486 2024-11-13T14:29:51,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191486) roll requested 2024-11-13T14:29:51,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191492 2024-11-13T14:29:51,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191492 2024-11-13T14:29:51,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191492) roll requested 2024-11-13T14:29:51,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191497 2024-11-13T14:29:51,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191497 2024-11-13T14:29:51,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191497) roll requested 2024-11-13T14:29:51,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191505 2024-11-13T14:29:51,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191505 2024-11-13T14:29:51,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191505) roll requested 2024-11-13T14:29:51,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191511 2024-11-13T14:29:51,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191511 2024-11-13T14:29:51,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191511) roll requested 2024-11-13T14:29:51,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191519 2024-11-13T14:29:51,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191519 2024-11-13T14:29:51,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191519) roll requested 2024-11-13T14:29:51,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191524 2024-11-13T14:29:51,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191524 2024-11-13T14:29:51,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191524) roll requested 2024-11-13T14:29:51,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191529 2024-11-13T14:29:51,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191529 2024-11-13T14:29:51,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191529) roll requested 2024-11-13T14:29:51,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191536 2024-11-13T14:29:51,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191536 2024-11-13T14:29:51,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191536) roll requested 2024-11-13T14:29:51,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191543 2024-11-13T14:29:51,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191543 2024-11-13T14:29:51,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191543) roll requested 2024-11-13T14:29:51,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191553 2024-11-13T14:29:51,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191553 2024-11-13T14:29:51,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191553) roll requested 2024-11-13T14:29:51,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191559 2024-11-13T14:29:51,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191559 2024-11-13T14:29:51,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191559) roll requested 2024-11-13T14:29:51,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191565 2024-11-13T14:29:51,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191565 2024-11-13T14:29:51,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191565) roll requested 2024-11-13T14:29:51,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191570 2024-11-13T14:29:51,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191570 2024-11-13T14:29:51,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191570) roll requested 2024-11-13T14:29:51,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191577 2024-11-13T14:29:51,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191577 2024-11-13T14:29:51,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191577) roll requested 2024-11-13T14:29:51,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191584 2024-11-13T14:29:51,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191584 2024-11-13T14:29:51,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191584) roll requested 2024-11-13T14:29:51,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191592 2024-11-13T14:29:51,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191592 2024-11-13T14:29:51,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191592) roll requested 2024-11-13T14:29:51,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191601 2024-11-13T14:29:51,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191601 2024-11-13T14:29:51,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191601) roll requested 2024-11-13T14:29:51,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191611 2024-11-13T14:29:51,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191611 2024-11-13T14:29:51,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191611) roll requested 2024-11-13T14:29:51,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191619 2024-11-13T14:29:51,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191619 2024-11-13T14:29:51,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191619) roll requested 2024-11-13T14:29:51,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191625 2024-11-13T14:29:51,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191625 2024-11-13T14:29:51,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191625) roll requested 2024-11-13T14:29:51,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191633 2024-11-13T14:29:51,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191633 2024-11-13T14:29:51,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191633) roll requested 2024-11-13T14:29:51,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191642 2024-11-13T14:29:51,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191642 2024-11-13T14:29:51,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191642) roll requested 2024-11-13T14:29:51,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191651 2024-11-13T14:29:51,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191651 2024-11-13T14:29:51,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191651) roll requested 2024-11-13T14:29:51,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191658 2024-11-13T14:29:51,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191658 2024-11-13T14:29:51,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191658) roll requested 2024-11-13T14:29:51,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191666 2024-11-13T14:29:51,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191666 2024-11-13T14:29:51,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191666) roll requested 2024-11-13T14:29:51,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191674 2024-11-13T14:29:51,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191674 2024-11-13T14:29:51,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191674) roll requested 2024-11-13T14:29:51,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191684 2024-11-13T14:29:51,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191684 2024-11-13T14:29:51,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191684) roll requested 2024-11-13T14:29:51,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191694 2024-11-13T14:29:51,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191694 2024-11-13T14:29:51,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191694) roll requested 2024-11-13T14:29:51,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191700 2024-11-13T14:29:51,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191700 2024-11-13T14:29:51,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191700) roll requested 2024-11-13T14:29:51,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191708 2024-11-13T14:29:51,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191708 2024-11-13T14:29:51,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191708) roll requested 2024-11-13T14:29:51,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191716 2024-11-13T14:29:51,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191716 2024-11-13T14:29:51,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191716) roll requested 2024-11-13T14:29:51,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191726 2024-11-13T14:29:51,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191726 2024-11-13T14:29:51,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191726) roll requested 2024-11-13T14:29:51,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191736 2024-11-13T14:29:51,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191736 2024-11-13T14:29:51,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191736) roll requested 2024-11-13T14:29:51,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191744 2024-11-13T14:29:51,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191744 2024-11-13T14:29:51,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191744) roll requested 2024-11-13T14:29:51,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191752 2024-11-13T14:29:51,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191752 2024-11-13T14:29:51,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191752) roll requested 2024-11-13T14:29:51,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191760 2024-11-13T14:29:51,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191760 2024-11-13T14:29:51,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191760) roll requested 2024-11-13T14:29:51,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191769 2024-11-13T14:29:51,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191769 2024-11-13T14:29:51,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191769) roll requested 2024-11-13T14:29:51,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191778 2024-11-13T14:29:51,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191778 2024-11-13T14:29:51,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191778) roll requested 2024-11-13T14:29:51,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191786 2024-11-13T14:29:51,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191786 2024-11-13T14:29:51,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191786) roll requested 2024-11-13T14:29:51,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191794 2024-11-13T14:29:51,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191794 2024-11-13T14:29:51,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191794) roll requested 2024-11-13T14:29:51,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191804 2024-11-13T14:29:51,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191804 2024-11-13T14:29:51,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191804) roll requested 2024-11-13T14:29:51,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191815 2024-11-13T14:29:51,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191815 2024-11-13T14:29:51,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191815) roll requested 2024-11-13T14:29:51,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191825 2024-11-13T14:29:51,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191825 2024-11-13T14:29:51,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191825) roll requested 2024-11-13T14:29:51,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191834 2024-11-13T14:29:51,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191834 2024-11-13T14:29:51,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191834) roll requested 2024-11-13T14:29:51,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191843 2024-11-13T14:29:51,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191843 2024-11-13T14:29:51,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191843) roll requested 2024-11-13T14:29:51,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191853 2024-11-13T14:29:51,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191853 2024-11-13T14:29:51,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191853) roll requested 2024-11-13T14:29:51,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191862 2024-11-13T14:29:51,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191862 2024-11-13T14:29:51,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191862) roll requested 2024-11-13T14:29:51,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191870 2024-11-13T14:29:51,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191870 2024-11-13T14:29:51,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191870) roll requested 2024-11-13T14:29:51,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191877 2024-11-13T14:29:51,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191877 2024-11-13T14:29:51,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191877) roll requested 2024-11-13T14:29:51,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191884 2024-11-13T14:29:51,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191884 2024-11-13T14:29:51,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191884) roll requested 2024-11-13T14:29:51,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191893 2024-11-13T14:29:51,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191893 2024-11-13T14:29:51,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191893) roll requested 2024-11-13T14:29:51,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191902 2024-11-13T14:29:51,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191902 2024-11-13T14:29:51,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191902) roll requested 2024-11-13T14:29:51,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191910 2024-11-13T14:29:51,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191910 2024-11-13T14:29:51,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191910) roll requested 2024-11-13T14:29:51,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191921 2024-11-13T14:29:51,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191921 2024-11-13T14:29:51,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191921) roll requested 2024-11-13T14:29:51,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191928 2024-11-13T14:29:51,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191928 2024-11-13T14:29:51,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191928) roll requested 2024-11-13T14:29:51,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191936 2024-11-13T14:29:51,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191936 2024-11-13T14:29:51,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191936) roll requested 2024-11-13T14:29:51,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191944 2024-11-13T14:29:51,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191944 2024-11-13T14:29:51,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191944) roll requested 2024-11-13T14:29:51,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191953 2024-11-13T14:29:51,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191953 2024-11-13T14:29:51,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191953) roll requested 2024-11-13T14:29:51,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191962 2024-11-13T14:29:51,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191962 2024-11-13T14:29:51,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191962) roll requested 2024-11-13T14:29:51,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191972 2024-11-13T14:29:51,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191972 2024-11-13T14:29:51,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191972) roll requested 2024-11-13T14:29:51,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191979 2024-11-13T14:29:51,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191979 2024-11-13T14:29:51,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191979) roll requested 2024-11-13T14:29:51,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191987 2024-11-13T14:29:51,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:51,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191987 2024-11-13T14:29:51,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:51,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:51,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:51,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:51,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191987) roll requested 2024-11-13T14:29:51,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508191996 2024-11-13T14:29:52,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191996 2024-11-13T14:29:52,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508191996) roll requested 2024-11-13T14:29:52,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192003 2024-11-13T14:29:52,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508191996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192003 2024-11-13T14:29:52,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192003) roll requested 2024-11-13T14:29:52,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192010 2024-11-13T14:29:52,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192010 2024-11-13T14:29:52,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192010) roll requested 2024-11-13T14:29:52,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192019 2024-11-13T14:29:52,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192019 2024-11-13T14:29:52,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192019) roll requested 2024-11-13T14:29:52,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192027 2024-11-13T14:29:52,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192027 2024-11-13T14:29:52,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192027) roll requested 2024-11-13T14:29:52,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192035 2024-11-13T14:29:52,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192035 2024-11-13T14:29:52,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192035) roll requested 2024-11-13T14:29:52,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192045 2024-11-13T14:29:52,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192045 2024-11-13T14:29:52,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192045) roll requested 2024-11-13T14:29:52,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192054 2024-11-13T14:29:52,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192054 2024-11-13T14:29:52,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192054) roll requested 2024-11-13T14:29:52,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192062 2024-11-13T14:29:52,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192062 2024-11-13T14:29:52,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192062) roll requested 2024-11-13T14:29:52,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192071 2024-11-13T14:29:52,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192071 2024-11-13T14:29:52,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192071) roll requested 2024-11-13T14:29:52,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192077 2024-11-13T14:29:52,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192077 2024-11-13T14:29:52,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192077) roll requested 2024-11-13T14:29:52,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192084 2024-11-13T14:29:52,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192084 2024-11-13T14:29:52,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192084) roll requested 2024-11-13T14:29:52,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192092 2024-11-13T14:29:52,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192092 2024-11-13T14:29:52,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192092) roll requested 2024-11-13T14:29:52,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192101 2024-11-13T14:29:52,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192101 2024-11-13T14:29:52,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192101) roll requested 2024-11-13T14:29:52,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192111 2024-11-13T14:29:52,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192111 2024-11-13T14:29:52,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192111) roll requested 2024-11-13T14:29:52,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192120 2024-11-13T14:29:52,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192120 2024-11-13T14:29:52,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192120) roll requested 2024-11-13T14:29:52,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192130 2024-11-13T14:29:52,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192130 2024-11-13T14:29:52,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192130) roll requested 2024-11-13T14:29:52,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192140 2024-11-13T14:29:52,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192140 2024-11-13T14:29:52,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192140) roll requested 2024-11-13T14:29:52,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192148 2024-11-13T14:29:52,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192148 2024-11-13T14:29:52,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192148) roll requested 2024-11-13T14:29:52,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192153 2024-11-13T14:29:52,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192153 2024-11-13T14:29:52,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192153) roll requested 2024-11-13T14:29:52,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192160 2024-11-13T14:29:52,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192160 2024-11-13T14:29:52,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192160) roll requested 2024-11-13T14:29:52,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192168 2024-11-13T14:29:52,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192168 2024-11-13T14:29:52,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192168) roll requested 2024-11-13T14:29:52,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192177 2024-11-13T14:29:52,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192177 2024-11-13T14:29:52,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192177) roll requested 2024-11-13T14:29:52,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192184 2024-11-13T14:29:52,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192184 2024-11-13T14:29:52,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192184) roll requested 2024-11-13T14:29:52,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192190 2024-11-13T14:29:52,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192190 2024-11-13T14:29:52,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192190) roll requested 2024-11-13T14:29:52,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192196 2024-11-13T14:29:52,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192196 2024-11-13T14:29:52,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192196) roll requested 2024-11-13T14:29:52,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192203 2024-11-13T14:29:52,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192203 2024-11-13T14:29:52,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192203) roll requested 2024-11-13T14:29:52,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192210 2024-11-13T14:29:52,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192210 2024-11-13T14:29:52,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192210) roll requested 2024-11-13T14:29:52,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192219 2024-11-13T14:29:52,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192219 2024-11-13T14:29:52,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192219) roll requested 2024-11-13T14:29:52,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192229 2024-11-13T14:29:52,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192229 2024-11-13T14:29:52,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192229) roll requested 2024-11-13T14:29:52,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192237 2024-11-13T14:29:52,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192237 2024-11-13T14:29:52,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192237) roll requested 2024-11-13T14:29:52,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192245 2024-11-13T14:29:52,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192245 2024-11-13T14:29:52,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192245) roll requested 2024-11-13T14:29:52,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192253 2024-11-13T14:29:52,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192253 2024-11-13T14:29:52,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192253) roll requested 2024-11-13T14:29:52,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192261 2024-11-13T14:29:52,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192261 2024-11-13T14:29:52,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192261) roll requested 2024-11-13T14:29:52,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192268 2024-11-13T14:29:52,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192268 2024-11-13T14:29:52,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T14:29:52,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731508192268) roll requested 2024-11-13T14:29:52,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731508192275 2024-11-13T14:29:52,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731508192275 2024-11-13T14:29:52,283 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:29:52,286 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing daa2ca368b2108264a9146503bae95e1, disabling compactions & flushes 2024-11-13T14:29:52,286 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1. 2024-11-13T14:29:52,287 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1. 2024-11-13T14:29:52,287 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1. after waiting 0 ms 2024-11-13T14:29:52,287 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1. 2024-11-13T14:29:52,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T14:29:52,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): daa2ca368b2108264a9146503bae95e1[MyCF] 2024-11-13T14:29:52,290 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1. 2024-11-13T14:29:52,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of daa2ca368b2108264a9146503bae95e1, because it is not online on us 2024-11-13T14:29:52,290 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T14:29:52,290 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1731508161972.daa2ca368b2108264a9146503bae95e1. 2024-11-13T14:29:52,290 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for daa2ca368b2108264a9146503bae95e1: Waiting for close lock at 1731508192286Disabling compacts and flushes for region at 1731508192286Disabling writes for close at 1731508192287 (+1 ms)Writing region close event to WAL at 1731508192290 (+3 ms)Closed at 1731508192290 2024-11-13T14:29:52,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T14:29:52,589 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1731508192275) 2024-11-13T14:29:52,589 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20e10038-c0e0-8515-c776-036ff235e6c9 2024-11-13T14:29:53,168 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=508 (was 692), ProcessCount=11 (was 11), AvailableMemoryMB=3517 (was 4168)